diff --git a/.flake8 b/.flake8 index d41ad4cd..9d6de545 100644 --- a/.flake8 +++ b/.flake8 @@ -25,6 +25,7 @@ exclude = venv, build, dist, + submodules, *.egg-info # Show source code for each error diff --git a/.gitmodules b/.gitmodules index 8662647b..3cef7753 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/nccl-tests"] path = submodules/nccl-tests url = git@github.com:NVIDIA/nccl-tests.git +[submodule "submodules/cuda-samples"] + path = submodules/cuda-samples + url = https://github.com/spike-zhu/cuda-samples.git diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..465ab506 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,90 @@ +# CUDA Compatibility Tests + +The compatibility adapter compiles and runs CUDA Samples on NVIDIA and +CUDA-compatible accelerator toolchains. It reports compilation, execution, +failure, and waived-sample counts without treating a partial pass rate as an +adapter execution error. + +## Test Input + +```json +{ + "run_id": "cuda_samples.nvidia.quick", + "testcase": "compatibility.CudaSamples.PassRate", + "config": { + "platform": "nvidia", + "sample_filter": ["vectorAdd", "matrixMul", "clock"], + "timeout_per_sample": 180, + "jobs": 4 + } +} +``` + +Initialize the bundled CUDA Samples revision before running compatibility tests: + +```bash +git submodule update --init submodules/cuda-samples +``` + +The adapter uses `submodules/cuda-samples` by default. An optional +`cuda_samples_dir` override must contain a `Samples` directory. The adapter +recursively discovers `Samples//` directories containing a +Makefile or a standalone CMake project. CMake grouping manifests that only +aggregate child directories are excluded. A requested sample name that is not +found, or an empty `sample_filter`, is a configuration error instead of a +successful zero-sample result. + +The bundled submodule pins the CMake-based `master` revision at `7b601789`. +The upstream `batch_test` branch provides the Makefiles used by the original +compatibility workflow. The adapter supports both layouts; use `build_system` +to select `cmake`, `make`, or the default `auto` detection. + +CMake samples are configured through a temporary wrapper project. The wrapper +sets `CUDA_ARCHITECTURES` on every generated target after the sample manifest +has been evaluated, so manifests that set their own default architecture list +cannot override the requested `sms` value. + +## Platform Toolchains + +Platform aliases and compiler candidates are shared with the hardware adapter +through `infinibench.hardware.constants`. Compatibility-only architecture +values and Make arguments live in `infinibench.common.constants`. + +| Platform | Default compiler | Default architecture | +| --- | --- | --- | +| NVIDIA | `nvcc` | `80` | +| MetaX | `cucc` (falls back to `mxcc`) | `70` | +| Iluvatar CoreX (BI-V150/TG150) | `/usr/local/corex/bin/clang++` | `ivcore11` | + +The supported canonical platform names are `cuda`, `metax`, and `corex`. +The existing aliases `nvidia` and `iluvatar` are also accepted. + +The CoreX default targets BI-V150/TG150. Override `sms` and `make_args` +together when testing a different Iluvatar architecture. + +Set `compiler`, `sms`, or `make_args` in the input when the installed vendor +SDK uses a wrapper or different target. Arguments are passed directly as an +argument list; shell expansion is not performed. The resolved default compiler +and architecture are added to the result config when they were not explicit in +the input. For MetaX, the adapter also infers `MACA_PATH` from the resolved +`cucc` or `mxcc` location when the variable is unset. It uses the cu-bridge +`cmake_maca` wrapper when available. An explicit `MACA_PATH` is preserved. + +For non-NVIDIA Makefile builds, the default arguments remove NVIDIA-only +`--threads`, `-gencode`, and `-m64` flags. Platform support is declared only +after its compile and runtime workflow has been validated on target hardware. + +## Metrics + +- `compile_passed` and `compile_failed` cover all discovered samples. +- `run_passed` and `run_failed` cover samples that produced an executable. +- `run_skipped` counts CUDA Samples that explicitly return a waived result. +- A sample that does not run because compilation failed has `run_result: + "not_run"` and is not included in `run_skipped`. +- `run_pass_rate` keeps the original end-to-end definition: run passes divided + by all selected samples. +- `details` records each sample path and the final compiler or runtime error. + +`result_code: 0` means the compatibility test completed and produced valid +measurements. It does not mean every sample passed; use the pass-rate metrics +for that decision. diff --git a/infinibench/common/constants.py b/infinibench/common/constants.py index e1e33079..64a992da 100644 --- a/infinibench/common/constants.py +++ b/infinibench/common/constants.py @@ -31,6 +31,7 @@ class TestCategory(str, Enum): INFER = "infer" COMM = "comm" TRAIN = "train" + COMPATIBILITY = "compatibility" # Valid test categories (derived from TestCategory enum) @@ -235,6 +236,43 @@ class InfiniCoreResult: } +# ============================================================ +# Compatibility Test Adapter Constants +# ============================================================ + +CUDA_SAMPLE_CONFIGS = { + "cuda": { + "sms": "80", + "make_args": (), + }, + "metax": { + "sms": "70", + "cmake_commands": ( + "/opt/maca/tools/cu-bridge/tools/cmake_maca", + "cmake_maca", + "cmake", + ), + "extra_env": {"CUCC_CMAKE_ENTRY": "2"}, + "make_args": ( + "ALL_CCFLAGS=--std=c++11", + "ALL_LDFLAGS=", + "GENCODE_FLAGS=", + ), + }, + "corex": { + "sms": "ivcore11", + "make_args": ( + "ALL_CCFLAGS=-x ivcore --cuda-gpu-arch=ivcore11 " + "--cuda-path=/usr/local/corex --std=c++11", + "ALL_LDFLAGS=--cuda-gpu-arch=ivcore11 " + "--cuda-path=/usr/local/corex -L/usr/local/corex/lib " + "-Wl,-rpath,/usr/local/corex/lib -lcudart", + "GENCODE_FLAGS=", + ), + }, +} + + # ============================================================ # Hardware Test Adapter Constants # ============================================================ diff --git a/infinibench/compatibility/__init__.py b/infinibench/compatibility/__init__.py new file mode 100644 index 00000000..5e03ceba --- /dev/null +++ b/infinibench/compatibility/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +"""Compatibility testing module for CUDA Samples compilation and execution.""" diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py new file mode 100644 index 00000000..b1829bba --- /dev/null +++ b/infinibench/compatibility/compatibility_adapter.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +"""CUDA Samples compatibility test adapter. + +Compiles and runs CUDA Samples on supported CUDA-compatible platforms and +reports compile/run pass rates. + +Testcase format: + compatibility.CudaSamples.PassRate +""" + +import logging +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from infinibench.adapter import BaseAdapter +from infinibench.common.constants import CUDA_SAMPLE_CONFIGS, ErrorCode, InfiniBenchJson +from infinibench.hardware.constants import PLATFORM_ALIASES, PLATFORM_CONFIGS +from infinibench.utils.time_utils import get_timestamp + +logger = logging.getLogger(__name__) +_METRIC_PREFIX = "compatibility.cuda_samples" + +# Repository submodule containing the pinned CUDA Samples revision. +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CUDA_SAMPLES_DIR = _REPO_ROOT / "submodules" / "cuda-samples" + + +class CompatibilityAdapter(BaseAdapter): + """Adapter for CUDA Samples compatibility tests.""" + + def process(self, test_input: Any) -> Dict[str, Any]: + """Process compatibility test.""" + test_dict = self._normalize_test_input(test_input) + if not test_dict: + return self._create_error_response( + "Invalid test input format", result_code=ErrorCode.CONFIG + ) + + testcase = test_dict.get(InfiniBenchJson.TESTCASE, "unknown") + config = test_dict.get(InfiniBenchJson.CONFIG, {}) + run_id = test_dict.get(InfiniBenchJson.RUN_ID, "unknown") + + logger.info(f"CompatibilityAdapter: Processing {testcase}") + + # Extract sub-test type from testcase (third component) + parts = testcase.split(".") + if len(parts) < 3: + return self._create_error_response( + f"Invalid testcase format: {testcase}. " + f"Expected: compatibility..", + test_dict, + result_code=ErrorCode.CONFIG, + ) + + if parts[1].lower() != "cudasamples": + return self._create_error_response( + f"Unknown compatibility sub-test: {parts[1].lower()}", + test_dict, + result_code=ErrorCode.CONFIG, + ) + + return { + InfiniBenchJson.RESULT_CODE: 0, + InfiniBenchJson.TIME: get_timestamp(), + InfiniBenchJson.RUN_ID: run_id, + InfiniBenchJson.TESTCASE: testcase, + InfiniBenchJson.CONFIG: config, + InfiniBenchJson.METRICS: self._run_cuda_samples_test(config), + } + + # ------------------------------------------------------------------ + # cuda-samples + # ------------------------------------------------------------------ + + def _run_cuda_samples_test(self, config: Dict[str, Any]) -> List[Dict]: + """Compile and run cuda-samples, collect pass rate.""" + requested_platform = str(config.get("platform", "cuda")).lower().strip() + platform = PLATFORM_ALIASES.get(requested_platform) + if not platform: + raise ValueError( + f"Unsupported CUDA-compatible platform: {requested_platform}" + ) + + samples_dir = config.get("cuda_samples_dir", str(_CUDA_SAMPLES_DIR)) + platform_config = CUDA_SAMPLE_CONFIGS.get(platform) + if platform_config is None: + raise KeyError( + f"CUDA Samples configuration not found for platform: {platform}" + ) + sms = self._validate_architectures(config.get("sms", platform_config["sms"])) + timeout_per_sample = self._positive_int( + config.get("timeout_per_sample", 60), "timeout_per_sample" + ) + jobs = self._positive_int(config.get("jobs", os.cpu_count() or 1), "jobs") + sample_filter = config.get("sample_filter") + build_system = str(config.get("build_system", "auto")).lower() + make_args = config.get("make_args") + if make_args is None: + make_args = list(platform_config["make_args"]) + elif not isinstance(make_args, list) or not all( + isinstance(argument, str) for argument in make_args + ): + raise ValueError("make_args must be a list of strings") + + samples_root = Path(samples_dir) / "Samples" + if not samples_root.exists(): + raise FileNotFoundError(f"CUDA samples directory not found: {samples_root}") + + sample_dirs = self._discover_sample_dirs(samples_root, sample_filter) + env = self._build_compile_env(platform, sms, config.get("compiler")) + config.setdefault("compiler", env["CUDACXX"]) + config.setdefault("sms", sms) + + details = [ + self._test_sample( + sample_dir, + samples_root, + env, + timeout_per_sample, + jobs, + build_system, + make_args, + ) + for sample_dir in sample_dirs + ] + return self._build_metrics(details) + + def _test_sample( + self, + sample_dir: Path, + samples_root: Path, + env: Dict[str, str], + timeout: int, + jobs: int, + build_system: str, + make_args: List[str], + ) -> Dict[str, Any]: + selected_build_system = self._select_build_system(sample_dir, build_system) + compile_result, run_result, error = "fail", "not_run", "" + + with tempfile.TemporaryDirectory( + prefix=f"infinibench-{sample_dir.name}-" + ) as temp_dir: + try: + binary = self._compile_sample( + sample_dir, + Path(temp_dir), + env, + timeout, + jobs, + selected_build_system, + make_args, + ) + compile_result = "pass" + run_result, error = self._run_sample(binary, timeout, env) + except subprocess.TimeoutExpired: + error = f"Compilation timed out after {timeout} seconds" + except Exception as exc: + if compile_result == "pass": + run_result = "fail" + error = str(exc) + finally: + if selected_build_system == "make": + self._clean_make_sample(sample_dir, env, timeout) + + return { + "name": sample_dir.name, + "path": sample_dir.relative_to(samples_root).as_posix(), + "compile_result": compile_result, + "run_result": run_result, + "error": error[-2000:] if error else "", + } + + @staticmethod + def _build_metrics(details: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + total = len(details) + if total == 0: + raise ValueError("No CUDA samples selected") + compile_passed = sum(d["compile_result"] == "pass" for d in details) + run_passed = sum(d["run_result"] == "pass" for d in details) + values = { + "total": total, + "compile_passed": compile_passed, + "compile_failed": total - compile_passed, + "compile_pass_rate": round(compile_passed / total * 100, 2), + "run_passed": run_passed, + "run_failed": sum(d["run_result"] == "fail" for d in details), + "run_skipped": sum(d["run_result"] == "skip" for d in details), + "run_pass_rate": round(run_passed / total * 100, 2), + } + metrics = [ + { + "name": f"{_METRIC_PREFIX}.{name}", + "value": value, + "type": "scalar", + "unit": "%" if name.endswith("pass_rate") else "", + } + for name, value in values.items() + ] + metrics.append( + { + "name": f"{_METRIC_PREFIX}.details", + "value": details, + "type": "detail", + "unit": "", + } + ) + return metrics + + @staticmethod + def _positive_int(value: Any, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a positive integer") from exc + if parsed <= 0: + raise ValueError(f"{name} must be a positive integer") + return parsed + + @staticmethod + def _validate_architectures(value: Any) -> str: + architectures = str(value).strip() + if not architectures or not re.fullmatch(r"[A-Za-z0-9_.+; -]+", architectures): + raise ValueError("sms contains invalid architecture characters") + return architectures + + @staticmethod + def _discover_sample_dirs( + samples_root: Path, sample_filter: Optional[List[str]] + ) -> List[Path]: + sample_dirs = sorted( + { + manifest.parent + for manifest_name in ("CMakeLists.txt", "Makefile") + for manifest in samples_root.rglob(manifest_name) + if len(manifest.parent.relative_to(samples_root).parts) >= 2 + and CompatibilityAdapter._is_standalone_manifest(manifest) + } + ) + if not sample_dirs: + raise ValueError(f"No CUDA samples found under: {samples_root}") + if sample_filter is None: + return sample_dirs + if ( + not isinstance(sample_filter, list) + or not sample_filter + or not all(isinstance(name, str) and name for name in sample_filter) + ): + raise ValueError("sample_filter must be a non-empty list of names") + + requested = set(sample_filter) + selected = [sample for sample in sample_dirs if sample.name in requested] + found = {sample.name for sample in selected} + missing = sorted(requested - found) + if missing: + raise ValueError(f"CUDA sample filters not found: {', '.join(missing)}") + return selected + + @staticmethod + def _is_standalone_manifest(manifest: Path) -> bool: + if manifest.name == "Makefile": + return True + try: + content = manifest.read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return ( + re.search(r"^\s*project\s*\(", content, re.IGNORECASE | re.MULTILINE) + is not None + ) + + @staticmethod + def _select_build_system(sample_dir: Path, requested: str) -> str: + if requested not in {"auto", "cmake", "make"}: + raise ValueError("build_system must be one of: auto, cmake, make") + if requested == "auto": + if (sample_dir / "CMakeLists.txt").exists(): + return "cmake" + if (sample_dir / "Makefile").exists(): + return "make" + raise ValueError(f"No supported build manifest in: {sample_dir}") + + manifest = "CMakeLists.txt" if requested == "cmake" else "Makefile" + if not (sample_dir / manifest).exists(): + raise ValueError(f"{manifest} not found in CUDA sample: {sample_dir}") + return requested + + def _build_compile_env( + self, platform: str, sms: str, compiler: Optional[str] = None + ) -> Dict[str, str]: + """Build environment variables for compilation.""" + env = os.environ.copy() + env["SMS"] = sms + + candidates = ( + (compiler,) if compiler else PLATFORM_CONFIGS[platform]["compilers"] + ) + compiler_path = next( + ( + resolved + for candidate in candidates + if (resolved := shutil.which(candidate, path=env.get("PATH"))) + ), + None, + ) + if not compiler_path: + raise FileNotFoundError( + f"No compiler found for platform {platform}; checked: " + + ", ".join(candidates) + ) + + env["CUDACXX"] = compiler_path + toolkit_root = str(Path(compiler_path).resolve().parent.parent) + env["CUDA_HOME"] = toolkit_root + env["CUDA_PATH"] = toolkit_root + if platform == "metax" and not env.get("MACA_PATH"): + maca_path = self._infer_metax_root(compiler_path) + if maca_path: + env["MACA_PATH"] = maca_path + + sample_config = CUDA_SAMPLE_CONFIGS[platform] + for name, value in sample_config.get("extra_env", {}).items(): + env.setdefault(name, value) + cmake_candidates = sample_config.get("cmake_commands", ("cmake",)) + cmake_command = next( + ( + resolved + for candidate in cmake_candidates + if (resolved := shutil.which(candidate, path=env.get("PATH"))) + ), + None, + ) + if cmake_command: + env["CMAKE_COMMAND"] = cmake_command + command_dir = str(Path(cmake_command).resolve().parent) + env["PATH"] = os.pathsep.join((command_dir, env.get("PATH", ""))) + + return env + + @staticmethod + def _infer_metax_root(compiler_path: str) -> Optional[str]: + path = Path(compiler_path).resolve() + if path.parts[-4:] == ("tools", "cu-bridge", "bin", "cucc"): + return str(path.parents[3]) + if path.parts[-3:] == ("mxgpu_llvm", "bin", "mxcc"): + return str(path.parents[2]) + return None + + def _compile_sample( + self, + sample_dir: Path, + build_dir: Path, + env: Dict[str, str], + timeout: int, + jobs: int, + build_system: str, + make_args: Optional[List[str]] = None, + ) -> Path: + """Compile one CUDA sample and return its executable.""" + sms = env.get("SMS", "80") + if build_system == "cmake": + cmake_command = env.get("CMAKE_COMMAND", "cmake") + wrapper_dir = build_dir / "source" + cmake_build_dir = build_dir / "build" + self._write_cmake_wrapper(wrapper_dir, sample_dir, sms) + cmake_build_dir.mkdir(parents=True, exist_ok=True) + configure_result = self._run_build_command( + [ + cmake_command, + "-S", + str(wrapper_dir), + "-B", + str(cmake_build_dir), + f"-DCMAKE_CUDA_ARCHITECTURES={sms}", + f"-DCMAKE_CUDA_COMPILER={env['CUDACXX']}", + ], + sample_dir, + env, + timeout, + ) + if configure_result.returncode: + raise RuntimeError(self._command_error(configure_result)) + command = [ + cmake_command, + "--build", + str(cmake_build_dir), + "--parallel", + str(jobs), + ] + search_root = cmake_build_dir + else: + self._run_build_command(["make", "clean"], sample_dir, env, timeout) + command = [ + "make", + f"-j{jobs}", + f"SMS={sms}", + f"NVCC={env['CUDACXX']}", + *(make_args or []), + ] + search_root = sample_dir + + build_result = self._run_build_command(command, sample_dir, env, timeout) + if build_result.returncode: + raise RuntimeError(self._command_error(build_result)) + + binary = self._find_sample_binary(search_root, sample_dir.name) + if not binary: + raise RuntimeError( + f"Build succeeded but no executable was found for {sample_dir.name}" + ) + return binary + + @staticmethod + def _write_cmake_wrapper(wrapper_dir: Path, sample_dir: Path, sms: str) -> None: + """Create an out-of-tree wrapper that owns the target architecture.""" + wrapper_dir.mkdir(parents=True, exist_ok=True) + sample_path = sample_dir.resolve().as_posix().replace('"', '\\"') + architecture = sms.replace('"', '\\"') + content = f"""cmake_minimum_required(VERSION 3.20) +project(InfiniBenchCudaSample LANGUAGES C CXX CUDA) + +add_subdirectory("{sample_path}" sample) + +function(infinibench_set_cuda_architectures directory) + get_property(targets DIRECTORY "${{directory}}" PROPERTY BUILDSYSTEM_TARGETS) + foreach(target IN LISTS targets) + get_target_property(target_type "${{target}}" TYPE) + if(NOT target_type STREQUAL "UTILITY" AND + NOT target_type STREQUAL "INTERFACE_LIBRARY") + set_property(TARGET "${{target}}" PROPERTY CUDA_ARCHITECTURES "{architecture}") + endif() + endforeach() + get_property(subdirectories DIRECTORY "${{directory}}" PROPERTY SUBDIRECTORIES) + foreach(subdirectory IN LISTS subdirectories) + infinibench_set_cuda_architectures("${{subdirectory}}") + endforeach() +endfunction() + +infinibench_set_cuda_architectures("{sample_path}") +""" + (wrapper_dir / "CMakeLists.txt").write_text(content, encoding="utf-8") + + @staticmethod + def _run_build_command( + command: List[str], cwd: Path, env: Dict[str, str], timeout: int + ) -> subprocess.CompletedProcess: + return subprocess.run( + command, + cwd=str(cwd), + capture_output=True, + text=True, + errors="replace", + env=env, + timeout=timeout, + ) + + @staticmethod + def _command_error(result: subprocess.CompletedProcess) -> str: + output = "\n".join(part for part in (result.stdout, result.stderr) if part) + return output[-2000:] or f"Command exited with code {result.returncode}" + + @staticmethod + def _find_sample_binary(search_root: Path, sample_name: str) -> Optional[Path]: + for name in (sample_name, sample_name.replace("_", "")): + direct = search_root / name + if direct.is_file() and os.access(direct, os.X_OK): + return direct + matches = sorted( + path + for path in search_root.rglob(name) + if path.is_file() and os.access(path, os.X_OK) + ) + if matches: + return matches[0] + return None + + def _run_sample( + self, binary: Path, timeout: int, env: Optional[Dict[str, str]] = None + ) -> Tuple[str, str]: + """Run a compiled CUDA sample.""" + binary = binary.resolve() + try: + result = subprocess.run( + [str(binary)], + cwd=str(binary.parent), + capture_output=True, + text=True, + errors="replace", + env=env, + timeout=timeout, + ) + return self._classify_run_result(result) + except subprocess.TimeoutExpired: + return "fail", f"Execution timed out after {timeout} seconds" + except FileNotFoundError as exc: + return "fail", str(exc) + + @staticmethod + def _classify_run_result( + result: subprocess.CompletedProcess, + ) -> Tuple[str, str]: + output = "\n".join(part for part in (result.stdout, result.stderr) if part) + normalized = output.lower() + if result.returncode == 2 or any( + marker in normalized + for marker in ("sample waived", "waiving sample", "result = waived") + ): + return "skip", output[-2000:] or "Sample waived" + if result.returncode == 0 and not re.search(r"result\s*=\s*fail", normalized): + return "pass", "" + return ( + "fail", + output[-2000:] or f"Executable exited with code {result.returncode}", + ) + + def _clean_make_sample( + self, sample_dir: Path, env: Dict[str, str], timeout: int + ) -> None: + try: + self._run_build_command(["make", "clean"], sample_dir, env, timeout) + except (FileNotFoundError, subprocess.TimeoutExpired): + logger.warning("Failed to clean CUDA sample build: %s", sample_dir) diff --git a/infinibench/dispatcher.py b/infinibench/dispatcher.py index a6870ef2..fb03eed9 100644 --- a/infinibench/dispatcher.py +++ b/infinibench/dispatcher.py @@ -24,6 +24,10 @@ (TestCategory.INFER, "vllm"): lambda: _create_inference_adapter(), (TestCategory.TRAIN, "megatron"): lambda: _create_training_adapter(), (TestCategory.TRAIN, "infinitrain"): lambda: _create_training_adapter(), + ( + TestCategory.COMPATIBILITY, + "cudasamples", + ): lambda: _create_compatibility_adapter(), } @@ -69,6 +73,13 @@ def _create_training_adapter(): return TrainingAdapter() +def _create_compatibility_adapter(): + """Create compatibility adapter (lazy import).""" + from infinibench.compatibility.compatibility_adapter import CompatibilityAdapter + + return CompatibilityAdapter() + + class Dispatcher: """Test orchestration dispatcher for managing test executions.""" diff --git a/infinibench/hardware/constants.py b/infinibench/hardware/constants.py index 03c6fb86..ec335621 100644 --- a/infinibench/hardware/constants.py +++ b/infinibench/hardware/constants.py @@ -23,41 +23,81 @@ "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "cuda", "cache_parser": "cuda", + "compilers": ("nvcc", "/usr/local/cuda/bin/nvcc"), + "detection_tools": ("nvcc", "nvidia-smi"), }, "metax": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "metax", "cache_parser": "cuda", + "compilers": ( + "/opt/maca/tools/cu-bridge/bin/cucc", + "cucc", + "/opt/maca/mxgpu_llvm/bin/mxcc", + "mxcc", + ), + "detection_tools": ( + "/opt/maca/tools/cu-bridge/bin/cucc", + "cucc", + "mxcc", + ), }, "corex": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "corex", "cache_parser": "cuda", + "compilers": ( + "/usr/local/corex/bin/clang++", + "/usr/local/corex/bin/nvcc", + "nvcc", + ), + "detection_tools": ( + "/usr/local/corex/bin/ixsmi", + "/usr/local/corex/bin/clang++", + ), }, "hygon": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "hygon", "cache_parser": "cuda", + "compilers": ("/opt/dtk/bin/hipcc", "hipcc"), + "detection_tools": ("/opt/dtk/bin/hy-smi", "hy-smi"), + "conditional_detection_tools": (("/opt/dtk", "hipcc"),), }, "moore": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "moore", "cache_parser": "cuda", + "compilers": ("/usr/local/musa/bin/mcc", "mcc"), + "detection_tools": ("mcc", "mthreads-gmi"), }, "cambricon": { "binary_name": "mlu_perf_suite", "benchmark_subdir": "cambricon-memory-benchmark", "build_platform": None, "cache_parser": "cambricon", + "detection_tools": ("cncc",), + "detection_paths": ("/usr/local/neuware",), }, "ascend": { "binary_name": "npu_perf_suite", "benchmark_subdir": "ascend-memory-benchmark", "build_platform": None, "cache_parser": "ascend", + "detection_tools": ("npu-smi", "atc"), + "detection_paths": ("/usr/local/Ascend/ascend-toolkit",), }, } + +PLATFORM_DETECTION_ORDER = ( + "ascend", + "cambricon", + "moore", + "metax", + "hygon", + "corex", +) diff --git a/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt b/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt index 223a6627..68bb8213 100644 --- a/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt +++ b/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt @@ -63,7 +63,7 @@ elseif(PLATFORM STREQUAL "corex") # CoreX adapted CMake handles CUDA compiler automatically (clang++) set(CMAKE_CUDA_STANDARD 17) set(CMAKE_CUDA_STANDARD_REQUIRED ON) - set(CMAKE_CUDA_ARCHITECTURES "ivcore20" CACHE STRING "CoreX GPU architectures") + set(CMAKE_CUDA_ARCHITECTURES "ivcore11" CACHE STRING "CoreX GPU architectures") include_directories($ENV{COREX_PATH}/include) link_directories($ENV{COREX_PATH}/lib64) diff --git a/infinibench/hardware/cuda-memory-benchmark/build.sh b/infinibench/hardware/cuda-memory-benchmark/build.sh index 723f3551..f6aaf1f5 100644 --- a/infinibench/hardware/cuda-memory-benchmark/build.sh +++ b/infinibench/hardware/cuda-memory-benchmark/build.sh @@ -110,7 +110,7 @@ elif [[ "$PLATFORM" == "corex" ]]; then echo -e "${YELLOW}Configuring with CoreX CMake...${NC}" cmake .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=corex \ - -DCMAKE_CUDA_ARCHITECTURES=ivcore20 + -DCMAKE_CUDA_ARCHITECTURES=ivcore11 echo -e "${YELLOW}Building...${NC}" make -j$(nproc) diff --git a/infinibench/hardware/hardware_adapter.py b/infinibench/hardware/hardware_adapter.py index 1b8c4dfb..a5a1e34a 100644 --- a/infinibench/hardware/hardware_adapter.py +++ b/infinibench/hardware/hardware_adapter.py @@ -25,7 +25,11 @@ InfiniBenchJson, ) from infinibench.common.csv_utils import create_timeseries_metric -from infinibench.hardware.constants import PLATFORM_ALIASES, PLATFORM_CONFIGS +from infinibench.hardware.constants import ( + PLATFORM_ALIASES, + PLATFORM_CONFIGS, + PLATFORM_DETECTION_ORDER, +) from infinibench.utils.time_utils import get_timestamp logger = logging.getLogger(__name__) @@ -33,39 +37,34 @@ def detect_platform() -> str: """Detect the installed accelerator toolchain.""" - if ( - shutil.which("npu-smi") - or shutil.which("atc") - or Path("/usr/local/Ascend/ascend-toolkit").exists() - ): - return "ascend" - if shutil.which("cncc") or Path("/usr/local/neuware").exists(): - return "cambricon" - if shutil.which("mcc") or shutil.which("mthreads-gmi"): - return "moore" - - maca_path = Path("/opt/maca") - if ( - (maca_path / "tools" / "cu-bridge" / "bin" / "cucc").exists() - or shutil.which("cucc") - or shutil.which("mxcc") - ): - return "metax" + for platform in PLATFORM_DETECTION_ORDER: + platform_config = PLATFORM_CONFIGS[platform] + tools = platform_config["detection_tools"] + if any(_tool_exists(tool) for tool in tools): + return platform + paths = platform_config.get("detection_paths", ()) + if any(_path_exists(path) for path in paths): + return platform + conditional_tools = platform_config.get("conditional_detection_tools", ()) + if any( + _path_exists(root) and _tool_exists(tool) + for root, tool in conditional_tools + ): + return platform + return "cuda" - dtk_path = Path("/opt/dtk") - if ( - (dtk_path / "bin" / "hy-smi").exists() - or shutil.which("hy-smi") - or (dtk_path.exists() and shutil.which("hipcc")) - ): - return "hygon" - corex_path = Path("/usr/local/corex") - if (corex_path / "bin" / "ixsmi").exists() or ( - corex_path / "bin" / "clang++" - ).exists(): - return "corex" - return "cuda" +def _tool_exists(tool: str) -> bool: + """Return whether a command or absolute tool path is available.""" + path = Path(tool) + if path.is_absolute(): + return path.is_file() + return shutil.which(tool) is not None + + +def _path_exists(path: str) -> bool: + """Return whether a platform-specific installation path exists.""" + return Path(path).exists() class HardwareTestAdapter(BaseAdapter): diff --git a/pyproject.toml b/pyproject.toml index a96214a2..d7996997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ exclude = ''' | buck-out | build | dist + | submodules | __pycache__ )/ ''' diff --git a/submodules/cuda-samples b/submodules/cuda-samples new file mode 160000 index 00000000..7b601789 --- /dev/null +++ b/submodules/cuda-samples @@ -0,0 +1 @@ +Subproject commit 7b60178984e96bc09d066077d5455df71fee2a9f diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py new file mode 100644 index 00000000..1aadaf95 --- /dev/null +++ b/tests/test_compatibility_adapter.py @@ -0,0 +1,399 @@ +import os +import subprocess +from pathlib import Path + +import pytest + +from infinibench.common.constants import CUDA_SAMPLE_CONFIGS +from infinibench.compatibility.compatibility_adapter import ( + _CUDA_SAMPLES_DIR, + CompatibilityAdapter, +) +from infinibench.dispatcher import Dispatcher + + +def _sample(root: Path, category: str, name: str, manifest: str) -> Path: + sample_dir = root / "Samples" / category / name + sample_dir.mkdir(parents=True) + content = f"project({name})\n" if manifest == "CMakeLists.txt" else "# test\n" + (sample_dir / manifest).write_text(content, encoding="utf-8") + return sample_dir + + +@pytest.fixture +def compiler_available(monkeypatch): + monkeypatch.setattr( + "infinibench.compatibility.compatibility_adapter.shutil.which", + lambda candidate, **_: candidate, + ) + + +def test_default_cuda_samples_dir_uses_repository_submodule(): + repository_root = Path(__file__).resolve().parents[1] + + assert _CUDA_SAMPLES_DIR == repository_root / "submodules" / "cuda-samples" + + +def test_discovers_nested_cmake_and_make_samples(tmp_path): + (tmp_path / "Samples").mkdir() + (tmp_path / "Samples" / "CMakeLists.txt").write_text("# aggregate\n") + category = tmp_path / "Samples" / "0_Introduction" + category.mkdir() + (category / "CMakeLists.txt").write_text("# aggregate\n") + vector_add = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + clock = _sample(tmp_path, "0_Introduction", "clock", "Makefile") + + discovered = CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", None + ) + + assert discovered == [clock, vector_add] + + +def test_sample_filter_rejects_unknown_names(tmp_path): + _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + + with pytest.raises(ValueError, match="missingSample"): + CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", ["vectorAdd", "missingSample"] + ) + + +def test_discovery_excludes_nested_cmake_group_manifests(tmp_path): + group = tmp_path / "Samples" / "8_Platform_Specific" / "Tegra" + group.mkdir(parents=True) + (group / "CMakeLists.txt").write_text( + "add_subdirectory(simpleGLES)\n", encoding="utf-8" + ) + sample = _sample( + tmp_path, + "8_Platform_Specific/Tegra", + "simpleGLES", + "CMakeLists.txt", + ) + + discovered = CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", None + ) + + assert discovered == [sample] + + +def test_sample_filter_rejects_an_empty_list(tmp_path): + _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + + with pytest.raises(ValueError, match="non-empty"): + CompatibilityAdapter()._discover_sample_dirs(tmp_path / "Samples", []) + + +@pytest.mark.parametrize("sms", ["", '80")\nmessage(FATAL_ERROR injected)']) +def test_architecture_rejects_empty_or_cmake_control_characters(sms): + with pytest.raises(ValueError, match="invalid architecture"): + CompatibilityAdapter._validate_architectures(sms) + + +@pytest.mark.parametrize( + ("build_system", "manifest", "sms", "compiler", "cmake_command", "jobs"), + [ + ( + "cmake", + "CMakeLists.txt", + "80", + "/usr/local/cuda/bin/nvcc", + "/vendor/bin/cmake_maca", + 7, + ), + ("make", "Makefile", "70", "/usr/local/musa/bin/mcc", "cmake", 3), + ], +) +def test_build_commands_include_compiler_arch_and_jobs( + tmp_path, + monkeypatch, + build_system, + manifest, + sms, + compiler, + cmake_command, + jobs, +): + sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", manifest) + build_dir = tmp_path / "build" + expected_binary = ( + build_dir / "build" if build_system == "cmake" else sample_dir + ) / "vectorAdd" + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + is_build = (len(command) > 1 and command[1] == "--build") or ( + command[0] == "make" and "clean" not in command + ) + if is_build: + expected_binary.parent.mkdir(parents=True, exist_ok=True) + expected_binary.write_text("binary", encoding="utf-8") + expected_binary.chmod(0o755) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + binary = CompatibilityAdapter()._compile_sample( + sample_dir, + build_dir, + { + "SMS": sms, + "CUDACXX": compiler, + "CMAKE_COMMAND": cmake_command, + }, + 30, + jobs, + build_system, + ) + + expected = ( + [ + cmake_command, + "--build", + str(build_dir / "build"), + "--parallel", + str(jobs), + ] + if build_system == "cmake" + else ["make", f"-j{jobs}", f"SMS={sms}", f"NVCC={compiler}"] + ) + assert binary == expected_binary + assert expected in calls + + +def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): + sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + wrapper_dir = tmp_path / "wrapper" + + CompatibilityAdapter._write_cmake_wrapper(wrapper_dir, sample_dir, "ivcore11") + + wrapper = (wrapper_dir / "CMakeLists.txt").read_text(encoding="utf-8") + assert f'add_subdirectory("{sample_dir.resolve().as_posix()}" sample)' in wrapper + assert 'PROPERTY CUDA_ARCHITECTURES "ivcore11"' in wrapper + + +@pytest.mark.parametrize("platform", ["metax", "corex"]) +def test_vendor_make_args_remove_nvidia_only_flags(platform): + args = " ".join(CUDA_SAMPLE_CONFIGS[platform]["make_args"]) + + assert all(flag not in args for flag in ("--threads", "-gencode", "-m64")) + + +def test_corex_make_args_keep_source_language_out_of_link_step(): + compile_args, link_args, _ = CUDA_SAMPLE_CONFIGS["corex"]["make_args"] + + assert "-x ivcore" in compile_args + assert "-x ivcore" not in link_args + + +def test_corex_defaults_target_bi_v150(): + config = CUDA_SAMPLE_CONFIGS["corex"] + compile_args, link_args, _ = config["make_args"] + + assert config["sms"] == "ivcore11" + assert "--cuda-gpu-arch=ivcore11" in compile_args + assert "--cuda-gpu-arch=ivcore11" in link_args + + +def test_compile_env_matches_selected_vendor_toolkit(monkeypatch, compiler_available): + compiler = "/usr/local/corex/bin/clang++" + monkeypatch.setenv("CUDA_HOME", "/usr/local/cuda") + monkeypatch.setenv("CUDA_PATH", "/usr/local/cuda") + env = CompatibilityAdapter()._build_compile_env("corex", "ivcore11", compiler) + + expected_root = str(Path(compiler).resolve().parent.parent) + assert env["CUDACXX"] == compiler + assert env["CUDA_HOME"] == expected_root + assert env["CUDA_PATH"] == expected_root + + +@pytest.mark.parametrize( + ("compiler", "root_parent_index"), + [ + ("/opt/maca/tools/cu-bridge/bin/cucc", 3), + ("/opt/maca/mxgpu_llvm/bin/mxcc", 2), + ], +) +def test_metax_compile_env_infers_maca_path( + monkeypatch, compiler_available, compiler, root_parent_index +): + monkeypatch.delenv("MACA_PATH", raising=False) + + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + assert env["MACA_PATH"] == str(Path(compiler).resolve().parents[root_parent_index]) + + +def test_metax_compile_env_preserves_explicit_maca_path( + monkeypatch, compiler_available +): + compiler = "/opt/maca/tools/cu-bridge/bin/cucc" + monkeypatch.setenv("MACA_PATH", "/custom/maca") + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + assert env["MACA_PATH"] == "/custom/maca" + + +def test_metax_compile_env_selects_cu_bridge_cmake(monkeypatch, compiler_available): + compiler = "/opt/maca/tools/cu-bridge/bin/cucc" + + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + cmake_command = "/opt/maca/tools/cu-bridge/tools/cmake_maca" + assert env["CMAKE_COMMAND"] == cmake_command + assert env["CUCC_CMAKE_ENTRY"] == "2" + assert env["PATH"].split(os.pathsep)[0] == str(Path(cmake_command).resolve().parent) + + +def test_cuda_sample_metrics_keep_skips_separate(tmp_path, monkeypatch): + samples = [ + _sample(tmp_path, "0_Introduction", name, "CMakeLists.txt") + for name in ("passes", "fails", "waived") + ] + binaries = {sample.name: tmp_path / f"{sample.name}.bin" for sample in samples} + adapter = CompatibilityAdapter() + + monkeypatch.setattr(adapter, "_discover_sample_dirs", lambda *_: samples) + monkeypatch.setattr( + adapter, + "_build_compile_env", + lambda *_: {"SMS": "80", "CUDACXX": "/usr/local/cuda/bin/nvcc"}, + ) + monkeypatch.setattr( + adapter, + "_compile_sample", + lambda sample_dir, *_: binaries[sample_dir.name], + ) + outcomes = { + "passes.bin": ("pass", ""), + "fails.bin": ("fail", "kernel failed"), + "waived.bin": ("skip", "sample waived"), + } + monkeypatch.setattr( + adapter, "_run_sample", lambda binary, *_: outcomes[binary.name] + ) + + config = {"platform": "nvidia", "cuda_samples_dir": str(tmp_path)} + metrics = adapter._run_cuda_samples_test(config) + values = {metric["name"]: metric["value"] for metric in metrics} + + assert config["compiler"] == "/usr/local/cuda/bin/nvcc" + assert config["sms"] == "80" + assert values["compatibility.cuda_samples.run_passed"] == 1 + assert values["compatibility.cuda_samples.run_failed"] == 1 + assert values["compatibility.cuda_samples.run_skipped"] == 1 + assert values["compatibility.cuda_samples.run_pass_rate"] == 33.33 + + +@pytest.mark.parametrize("failure_stage", ["compile", "run"]) +def test_sample_failures_are_accounted_for(tmp_path, monkeypatch, failure_stage): + sample = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + adapter = CompatibilityAdapter() + monkeypatch.setattr(adapter, "_discover_sample_dirs", lambda *_: [sample]) + monkeypatch.setattr( + adapter, + "_build_compile_env", + lambda *_: {"SMS": "80", "CUDACXX": "/usr/local/cuda/bin/nvcc"}, + ) + + def compile_sample(*_): + if failure_stage == "compile": + raise RuntimeError("compiler failed") + return tmp_path / "vectorAdd" + + def run_sample(*_): + raise RuntimeError("runtime failed") + + monkeypatch.setattr(adapter, "_compile_sample", compile_sample) + monkeypatch.setattr(adapter, "_run_sample", run_sample) + + metrics = adapter._run_cuda_samples_test( + {"platform": "nvidia", "cuda_samples_dir": str(tmp_path)} + ) + values = {metric["name"]: metric["value"] for metric in metrics} + details = values["compatibility.cuda_samples.details"] + + compile_passed = int(failure_stage == "run") + assert values["compatibility.cuda_samples.compile_passed"] == compile_passed + assert values["compatibility.cuda_samples.run_failed"] == compile_passed + assert values["compatibility.cuda_samples.run_skipped"] == 0 + assert details[0]["compile_result"] == ("pass" if compile_passed else "fail") + assert details[0]["run_result"] == ("fail" if compile_passed else "not_run") + + +def test_run_sample_classifies_cuda_waiver(tmp_path): + binary = tmp_path / "sample" + binary.write_text("binary", encoding="utf-8") + binary.chmod(0o755) + adapter = CompatibilityAdapter() + + result, error = adapter._classify_run_result( + subprocess.CompletedProcess([str(binary)], 2, "Sample waived", "") + ) + + assert result == "skip" + assert "waived" in error.lower() + + +def test_run_sample_uses_the_binary_directory(tmp_path, monkeypatch): + binary = tmp_path / "build" / "sample" + binary.parent.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, "Result = PASS", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result, _ = CompatibilityAdapter()._run_sample(binary, 30) + + assert result == "pass" + assert calls == [ + ( + [str(binary)], + { + "cwd": str(binary.parent), + "capture_output": True, + "text": True, + "errors": "replace", + "env": None, + "timeout": 30, + }, + ) + ] + + +def test_run_sample_resolves_a_relative_binary_path(tmp_path, monkeypatch): + binary = Path("build") / "sample" + monkeypatch.chdir(tmp_path) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs["cwd"])) + return subprocess.CompletedProcess(command, 0, "Result = PASS", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result, _ = CompatibilityAdapter()._run_sample(binary, 30) + + resolved_binary = binary.resolve() + assert result == "pass" + assert calls == [([str(resolved_binary)], str(resolved_binary.parent))] + + +def test_metrics_reject_an_empty_sample_set(): + with pytest.raises(ValueError, match="No CUDA samples selected"): + CompatibilityAdapter._build_metrics([]) + + +def test_dispatcher_only_registers_cuda_samples_compatibility(): + assert isinstance( + Dispatcher()._create_adapter("compatibility", "cudasamples"), + CompatibilityAdapter, + ) + for framework in ("megatron", "vllm", "infinilm"): + with pytest.raises(ValueError, match="Adapter not registered"): + Dispatcher()._create_adapter("compatibility", framework) diff --git a/tests/test_hardware_adapter.py b/tests/test_hardware_adapter.py index 585d9afd..e373a2cc 100644 --- a/tests/test_hardware_adapter.py +++ b/tests/test_hardware_adapter.py @@ -4,7 +4,7 @@ from infinibench.dispatcher import Dispatcher from infinibench.hardware import hardware_adapter -from infinibench.hardware.hardware_adapter import HardwareTestAdapter +from infinibench.hardware.hardware_adapter import HardwareTestAdapter, detect_platform CUDA_OUTPUT = """ Direction: Host to Device @@ -178,6 +178,54 @@ def test_runtime_detection_ignores_testcase_framework(tmp_path, monkeypatch): assert adapter._get_device_type({}) == "moore" +def test_runtime_detection_uses_shared_detection_tools(monkeypatch): + monkeypatch.setattr( + hardware_adapter, + "_tool_exists", + lambda tool: tool == "/opt/maca/tools/cu-bridge/bin/cucc", + ) + + assert detect_platform() == "metax" + + +def test_runtime_detection_accepts_ascend_toolkit_path(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda *_: False) + monkeypatch.setattr( + hardware_adapter, + "_path_exists", + lambda path: path == "/usr/local/Ascend/ascend-toolkit", + ) + + assert detect_platform() == "ascend" + + +def test_runtime_detection_accepts_cambricon_toolkit_path(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda *_: False) + monkeypatch.setattr( + hardware_adapter, + "_path_exists", + lambda path: path == "/usr/local/neuware", + ) + + assert detect_platform() == "cambricon" + + +def test_runtime_detection_does_not_treat_rocm_hipcc_as_hygon(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda tool: tool == "hipcc") + monkeypatch.setattr(hardware_adapter, "_path_exists", lambda *_: False) + + assert detect_platform() == "cuda" + + +def test_runtime_detection_accepts_hipcc_inside_dtk(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda tool: tool == "hipcc") + monkeypatch.setattr( + hardware_adapter, "_path_exists", lambda path: path == "/opt/dtk" + ) + + assert detect_platform() == "hygon" + + @pytest.mark.parametrize("device", ["cuda", "metax", "corex", "hygon", "moore"]) def test_cuda_compatible_platforms_share_binary(tmp_path, device): cuda_binary = tmp_path / "cuda_perf_suite" diff --git a/tests/test_hardware_detection.py b/tests/test_hardware_detection.py index 3878a8b4..1ef5c982 100644 --- a/tests/test_hardware_detection.py +++ b/tests/test_hardware_detection.py @@ -5,6 +5,7 @@ from infinibench.utils import hardware_detector from infinibench.utils.hardware_detector import HardwareDetector + MTHREADS_OUTPUT = """ Attached GPUs : 2