From 3ddc4e2867ac5f440cd7cbfa571072d637100431 Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Thu, 6 Aug 2026 23:25:42 +0800 Subject: [PATCH 1/7] feat: add CUDA Samples compatibility testing --- docs/compatibility.md | 82 +++ infinibench/common/constants.py | 32 ++ infinibench/compatibility/__init__.py | 2 + .../compatibility/compatibility_adapter.py | 510 ++++++++++++++++++ infinibench/dispatcher.py | 11 + infinibench/hardware/constants.py | 40 ++ infinibench/hardware/hardware_adapter.py | 63 ++- tests/test_compatibility_adapter.py | 329 +++++++++++ tests/test_hardware_adapter.py | 50 +- tests/test_hardware_detection.py | 1 + 10 files changed, 1087 insertions(+), 33 deletions(-) create mode 100644 docs/compatibility.md create mode 100644 infinibench/compatibility/__init__.py create mode 100644 infinibench/compatibility/compatibility_adapter.py create mode 100644 tests/test_compatibility_adapter.py diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..d9214458 --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,82 @@ +# CUDA Compatibility Tests + +The compatibility adapter compiles and runs CUDA Samples on NVIDIA and +CUDA-compatible accelerator toolchains. It reports compilation, execution, +failure, and waived-sample counts without treating a partial pass rate as an +adapter execution error. + +## Test Input + +```json +{ + "run_id": "cuda_samples.nvidia.quick", + "testcase": "compatibility.CudaSamples.PassRate", + "config": { + "platform": "nvidia", + "cuda_samples_dir": "/workspace/cuda-samples", + "build_system": "make", + "sample_filter": ["vectorAdd", "matrixMul", "clock"], + "timeout_per_sample": 180, + "jobs": 4 + } +} +``` + +`cuda_samples_dir` must contain a `Samples` directory. The adapter recursively +discovers `Samples//` directories containing a Makefile or a +standalone CMake project. CMake grouping manifests that only aggregate child +directories are excluded. A requested sample name that is not found, or an +empty `sample_filter`, is a configuration error instead of a successful +zero-sample result. + +The InfiniPerf cuda-samples submodule pins the CMake-based `master` revision. +Its `batch_test` branch provides the Makefiles used by the original InfiniPerf +compatibility workflow. The adapter supports both layouts; use `build_system` +to select `cmake`, `make`, or the default `auto` detection. + +CMake samples are configured through a temporary wrapper project. The wrapper +sets `CUDA_ARCHITECTURES` on every generated target after the sample manifest +has been evaluated, so manifests that set their own default architecture list +cannot override the requested `sms` value. + +## Platform Toolchains + +Platform aliases and compiler candidates are shared with the hardware adapter +through `infinibench.hardware.constants`. Compatibility-only architecture +values and Make arguments live in `infinibench.common.constants`. + +| Platform | Default compiler | Default architecture | +| --- | --- | --- | +| NVIDIA | `nvcc` | `80` | +| MetaX | `cucc` (falls back to `mxcc`) | `70` | +| Iluvatar CoreX | `/usr/local/corex/bin/clang++` | `ivcore20` | + +The supported canonical platform names are `cuda`, `metax`, and `corex`. +The existing aliases `nvidia` and `iluvatar` are also accepted. + +Set `compiler`, `sms`, or `make_args` in the input when the installed vendor +SDK uses a wrapper or different target. Arguments are passed directly as an +argument list; shell expansion is not performed. The resolved default compiler +and architecture are added to the result config when they were not explicit in +the input. For MetaX, the adapter also infers `MACA_PATH` from the resolved +`cucc` or `mxcc` location when the variable is unset. An explicit `MACA_PATH` +is preserved. + +For non-NVIDIA Makefile builds, the default arguments remove NVIDIA-only +`--threads`, `-gencode`, and `-m64` flags. Platform support is declared only +after its compile and runtime workflow has been validated on target hardware. + +## Metrics + +- `compile_passed` and `compile_failed` cover all discovered samples. +- `run_passed` and `run_failed` cover samples that produced an executable. +- `run_skipped` counts CUDA Samples that explicitly return a waived result. +- A sample that does not run because compilation failed has `run_result: + "not_run"` and is not included in `run_skipped`. +- `run_pass_rate` keeps the original end-to-end definition: run passes divided + by all selected samples. +- `details` records each sample path and the final compiler or runtime error. + +`result_code: 0` means the compatibility test completed and produced valid +measurements. It does not mean every sample passed; use the pass-rate metrics +for that decision. diff --git a/infinibench/common/constants.py b/infinibench/common/constants.py index e1e33079..5bcdaab1 100644 --- a/infinibench/common/constants.py +++ b/infinibench/common/constants.py @@ -31,6 +31,7 @@ class TestCategory(str, Enum): INFER = "infer" COMM = "comm" TRAIN = "train" + COMPATIBILITY = "compatibility" # Valid test categories (derived from TestCategory enum) @@ -235,6 +236,37 @@ class InfiniCoreResult: } +# ============================================================ +# Compatibility Test Adapter Constants +# ============================================================ + +CUDA_SAMPLE_CONFIGS = { + "cuda": { + "sms": "80", + "make_args": (), + }, + "metax": { + "sms": "70", + "make_args": ( + "ALL_CCFLAGS=--std=c++11", + "ALL_LDFLAGS=", + "GENCODE_FLAGS=", + ), + }, + "corex": { + "sms": "ivcore20", + "make_args": ( + "ALL_CCFLAGS=-x ivcore --cuda-gpu-arch=ivcore20 " + "--cuda-path=/usr/local/corex --std=c++11", + "ALL_LDFLAGS=--cuda-gpu-arch=ivcore20 " + "--cuda-path=/usr/local/corex -L/usr/local/corex/lib " + "-Wl,-rpath,/usr/local/corex/lib -lcudart", + "GENCODE_FLAGS=", + ), + }, +} + + # ============================================================ # Hardware Test Adapter Constants # ============================================================ diff --git a/infinibench/compatibility/__init__.py b/infinibench/compatibility/__init__.py new file mode 100644 index 00000000..5e03ceba --- /dev/null +++ b/infinibench/compatibility/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +"""Compatibility testing module for CUDA Samples compilation and execution.""" diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py new file mode 100644 index 00000000..5440db52 --- /dev/null +++ b/infinibench/compatibility/compatibility_adapter.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python3 +"""CUDA Samples compatibility test adapter. + +Compiles and runs CUDA Samples on supported CUDA-compatible platforms and +reports compile/run pass rates. + +Testcase format: + compatibility.CudaSamples.PassRate +""" + +import logging +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from infinibench.adapter import BaseAdapter +from infinibench.common.constants import CUDA_SAMPLE_CONFIGS, ErrorCode, InfiniBenchJson +from infinibench.hardware.constants import PLATFORM_ALIASES, PLATFORM_CONFIGS +from infinibench.utils.time_utils import get_timestamp + +logger = logging.getLogger(__name__) +_METRIC_PREFIX = "compatibility.cuda_samples" + +# Repo root for finding cuda-samples +_REPO_ROOT = Path(__file__).resolve().parents[3] +_CUDA_SAMPLES_DIR = ( + _REPO_ROOT / "InfiniPerf" / "benchmarks" / "compatibility" / "cuda-samples" +) + + +class CompatibilityAdapter(BaseAdapter): + """Adapter for CUDA Samples compatibility tests.""" + + def process(self, test_input: Any) -> Dict[str, Any]: + """Process compatibility test.""" + test_dict = self._normalize_test_input(test_input) + if not test_dict: + return self._create_error_response( + "Invalid test input format", result_code=ErrorCode.CONFIG + ) + + testcase = test_dict.get(InfiniBenchJson.TESTCASE, "unknown") + config = test_dict.get(InfiniBenchJson.CONFIG, {}) + run_id = test_dict.get(InfiniBenchJson.RUN_ID, "unknown") + + logger.info(f"CompatibilityAdapter: Processing {testcase}") + + # Extract sub-test type from testcase (third component) + parts = testcase.split(".") + if len(parts) < 3: + return self._create_error_response( + f"Invalid testcase format: {testcase}. " + f"Expected: compatibility..", + test_dict, + result_code=ErrorCode.CONFIG, + ) + + if parts[1].lower() != "cudasamples": + return self._create_error_response( + f"Unknown compatibility sub-test: {parts[1].lower()}", + test_dict, + result_code=ErrorCode.CONFIG, + ) + + return { + InfiniBenchJson.RESULT_CODE: 0, + InfiniBenchJson.TIME: get_timestamp(), + InfiniBenchJson.RUN_ID: run_id, + InfiniBenchJson.TESTCASE: testcase, + InfiniBenchJson.CONFIG: config, + InfiniBenchJson.METRICS: self._run_cuda_samples_test(config), + } + + # ------------------------------------------------------------------ + # cuda-samples + # ------------------------------------------------------------------ + + def _run_cuda_samples_test(self, config: Dict[str, Any]) -> List[Dict]: + """Compile and run cuda-samples, collect pass rate.""" + requested_platform = str(config.get("platform", "cuda")).lower().strip() + platform = PLATFORM_ALIASES.get(requested_platform) + if not platform: + raise ValueError( + f"Unsupported CUDA-compatible platform: {requested_platform}" + ) + + samples_dir = config.get("cuda_samples_dir", str(_CUDA_SAMPLES_DIR)) + platform_config = CUDA_SAMPLE_CONFIGS.get(platform) + if platform_config is None: + raise KeyError( + f"CUDA Samples configuration not found for platform: {platform}" + ) + sms = self._validate_architectures(config.get("sms", platform_config["sms"])) + timeout_per_sample = self._positive_int( + config.get("timeout_per_sample", 60), "timeout_per_sample" + ) + jobs = self._positive_int(config.get("jobs", os.cpu_count() or 1), "jobs") + sample_filter = config.get("sample_filter") + build_system = str(config.get("build_system", "auto")).lower() + make_args = config.get("make_args") + if make_args is None: + make_args = list(platform_config["make_args"]) + elif not isinstance(make_args, list) or not all( + isinstance(argument, str) for argument in make_args + ): + raise ValueError("make_args must be a list of strings") + + samples_root = Path(samples_dir) / "Samples" + if not samples_root.exists(): + raise FileNotFoundError(f"CUDA samples directory not found: {samples_root}") + + sample_dirs = self._discover_sample_dirs(samples_root, sample_filter) + env = self._build_compile_env(platform, sms, config.get("compiler")) + config.setdefault("compiler", env["CUDACXX"]) + config.setdefault("sms", sms) + + details = [ + self._test_sample( + sample_dir, + samples_root, + env, + timeout_per_sample, + jobs, + build_system, + make_args, + ) + for sample_dir in sample_dirs + ] + return self._build_metrics(details) + + def _test_sample( + self, + sample_dir: Path, + samples_root: Path, + env: Dict[str, str], + timeout: int, + jobs: int, + build_system: str, + make_args: List[str], + ) -> Dict[str, Any]: + selected_build_system = self._select_build_system(sample_dir, build_system) + compile_result, run_result, error = "fail", "not_run", "" + + with tempfile.TemporaryDirectory( + prefix=f"infinibench-{sample_dir.name}-" + ) as temp_dir: + try: + binary = self._compile_sample( + sample_dir, + Path(temp_dir), + env, + timeout, + jobs, + selected_build_system, + make_args, + ) + compile_result = "pass" + run_result, error = self._run_sample(binary, timeout, env) + except subprocess.TimeoutExpired: + error = f"Compilation timed out after {timeout} seconds" + except Exception as exc: + if compile_result == "pass": + run_result = "fail" + error = str(exc) + finally: + if selected_build_system == "make": + self._clean_make_sample(sample_dir, env, timeout) + + return { + "name": sample_dir.name, + "path": sample_dir.relative_to(samples_root).as_posix(), + "compile_result": compile_result, + "run_result": run_result, + "error": error[-2000:] if error else "", + } + + @staticmethod + def _build_metrics(details: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + total = len(details) + if total == 0: + raise ValueError("No CUDA samples selected") + compile_passed = sum(d["compile_result"] == "pass" for d in details) + run_passed = sum(d["run_result"] == "pass" for d in details) + values = { + "total": total, + "compile_passed": compile_passed, + "compile_failed": total - compile_passed, + "compile_pass_rate": round(compile_passed / total * 100, 2), + "run_passed": run_passed, + "run_failed": sum(d["run_result"] == "fail" for d in details), + "run_skipped": sum(d["run_result"] == "skip" for d in details), + "run_pass_rate": round(run_passed / total * 100, 2), + } + metrics = [ + { + "name": f"{_METRIC_PREFIX}.{name}", + "value": value, + "type": "scalar", + "unit": "%" if name.endswith("pass_rate") else "", + } + for name, value in values.items() + ] + metrics.append( + { + "name": f"{_METRIC_PREFIX}.details", + "value": details, + "type": "detail", + "unit": "", + } + ) + return metrics + + @staticmethod + def _positive_int(value: Any, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a positive integer") from exc + if parsed <= 0: + raise ValueError(f"{name} must be a positive integer") + return parsed + + @staticmethod + def _validate_architectures(value: Any) -> str: + architectures = str(value).strip() + if not architectures or not re.fullmatch(r"[A-Za-z0-9_.+; -]+", architectures): + raise ValueError("sms contains invalid architecture characters") + return architectures + + @staticmethod + def _discover_sample_dirs( + samples_root: Path, sample_filter: Optional[List[str]] + ) -> List[Path]: + sample_dirs = sorted( + { + manifest.parent + for manifest_name in ("CMakeLists.txt", "Makefile") + for manifest in samples_root.rglob(manifest_name) + if len(manifest.parent.relative_to(samples_root).parts) >= 2 + and CompatibilityAdapter._is_standalone_manifest(manifest) + } + ) + if not sample_dirs: + raise ValueError(f"No CUDA samples found under: {samples_root}") + if sample_filter is None: + return sample_dirs + if ( + not isinstance(sample_filter, list) + or not sample_filter + or not all(isinstance(name, str) and name for name in sample_filter) + ): + raise ValueError("sample_filter must be a non-empty list of names") + + requested = set(sample_filter) + selected = [sample for sample in sample_dirs if sample.name in requested] + found = {sample.name for sample in selected} + missing = sorted(requested - found) + if missing: + raise ValueError(f"CUDA sample filters not found: {', '.join(missing)}") + return selected + + @staticmethod + def _is_standalone_manifest(manifest: Path) -> bool: + if manifest.name == "Makefile": + return True + try: + content = manifest.read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return ( + re.search(r"^\s*project\s*\(", content, re.IGNORECASE | re.MULTILINE) + is not None + ) + + @staticmethod + def _select_build_system(sample_dir: Path, requested: str) -> str: + if requested not in {"auto", "cmake", "make"}: + raise ValueError("build_system must be one of: auto, cmake, make") + if requested == "auto": + if (sample_dir / "CMakeLists.txt").exists(): + return "cmake" + if (sample_dir / "Makefile").exists(): + return "make" + raise ValueError(f"No supported build manifest in: {sample_dir}") + + manifest = "CMakeLists.txt" if requested == "cmake" else "Makefile" + if not (sample_dir / manifest).exists(): + raise ValueError(f"{manifest} not found in CUDA sample: {sample_dir}") + return requested + + def _build_compile_env( + self, platform: str, sms: str, compiler: Optional[str] = None + ) -> Dict[str, str]: + """Build environment variables for compilation.""" + env = os.environ.copy() + env["SMS"] = sms + + candidates = ( + (compiler,) if compiler else PLATFORM_CONFIGS[platform]["compilers"] + ) + compiler_path = next( + ( + resolved + for candidate in candidates + if (resolved := shutil.which(candidate, path=env.get("PATH"))) + ), + None, + ) + if not compiler_path: + raise FileNotFoundError( + f"No compiler found for platform {platform}; checked: " + + ", ".join(candidates) + ) + + env["CUDACXX"] = compiler_path + toolkit_root = str(Path(compiler_path).resolve().parent.parent) + env["CUDA_HOME"] = toolkit_root + env["CUDA_PATH"] = toolkit_root + if platform == "metax" and not env.get("MACA_PATH"): + maca_path = self._infer_metax_root(compiler_path) + if maca_path: + env["MACA_PATH"] = maca_path + + return env + + @staticmethod + def _infer_metax_root(compiler_path: str) -> Optional[str]: + path = Path(compiler_path).resolve() + if path.parts[-4:] == ("tools", "cu-bridge", "bin", "cucc"): + return str(path.parents[3]) + if path.parts[-3:] == ("mxgpu_llvm", "bin", "mxcc"): + return str(path.parents[2]) + return None + + def _compile_sample( + self, + sample_dir: Path, + build_dir: Path, + env: Dict[str, str], + timeout: int, + jobs: int, + build_system: str, + make_args: Optional[List[str]] = None, + ) -> Path: + """Compile one CUDA sample and return its executable.""" + sms = env.get("SMS", "80") + if build_system == "cmake": + wrapper_dir = build_dir / "source" + cmake_build_dir = build_dir / "build" + self._write_cmake_wrapper(wrapper_dir, sample_dir, sms) + cmake_build_dir.mkdir(parents=True, exist_ok=True) + configure_result = self._run_build_command( + [ + "cmake", + "-S", + str(wrapper_dir), + "-B", + str(cmake_build_dir), + f"-DCMAKE_CUDA_ARCHITECTURES={sms}", + f"-DCMAKE_CUDA_COMPILER={env['CUDACXX']}", + ], + sample_dir, + env, + timeout, + ) + if configure_result.returncode: + raise RuntimeError(self._command_error(configure_result)) + command = [ + "cmake", + "--build", + str(cmake_build_dir), + "--parallel", + str(jobs), + ] + search_root = cmake_build_dir + else: + self._run_build_command(["make", "clean"], sample_dir, env, timeout) + command = [ + "make", + f"-j{jobs}", + f"SMS={sms}", + f"NVCC={env['CUDACXX']}", + *(make_args or []), + ] + search_root = sample_dir + + build_result = self._run_build_command(command, sample_dir, env, timeout) + if build_result.returncode: + raise RuntimeError(self._command_error(build_result)) + + binary = self._find_sample_binary(search_root, sample_dir.name) + if not binary: + raise RuntimeError( + f"Build succeeded but no executable was found for {sample_dir.name}" + ) + return binary + + @staticmethod + def _write_cmake_wrapper(wrapper_dir: Path, sample_dir: Path, sms: str) -> None: + """Create an out-of-tree wrapper that owns the target architecture.""" + wrapper_dir.mkdir(parents=True, exist_ok=True) + sample_path = sample_dir.resolve().as_posix().replace('"', '\\"') + architecture = sms.replace('"', '\\"') + content = f"""cmake_minimum_required(VERSION 3.20) +project(InfiniBenchCudaSample LANGUAGES C CXX CUDA) + +add_subdirectory("{sample_path}" sample) + +function(infinibench_set_cuda_architectures directory) + get_property(targets DIRECTORY "${{directory}}" PROPERTY BUILDSYSTEM_TARGETS) + foreach(target IN LISTS targets) + get_target_property(target_type "${{target}}" TYPE) + if(NOT target_type STREQUAL "UTILITY" AND + NOT target_type STREQUAL "INTERFACE_LIBRARY") + set_property(TARGET "${{target}}" PROPERTY CUDA_ARCHITECTURES "{architecture}") + endif() + endforeach() + get_property(subdirectories DIRECTORY "${{directory}}" PROPERTY SUBDIRECTORIES) + foreach(subdirectory IN LISTS subdirectories) + infinibench_set_cuda_architectures("${{subdirectory}}") + endforeach() +endfunction() + +infinibench_set_cuda_architectures("{sample_path}") +""" + (wrapper_dir / "CMakeLists.txt").write_text(content, encoding="utf-8") + + @staticmethod + def _run_build_command( + command: List[str], cwd: Path, env: Dict[str, str], timeout: int + ) -> subprocess.CompletedProcess: + return subprocess.run( + command, + cwd=str(cwd), + capture_output=True, + text=True, + errors="replace", + env=env, + timeout=timeout, + ) + + @staticmethod + def _command_error(result: subprocess.CompletedProcess) -> str: + output = "\n".join(part for part in (result.stdout, result.stderr) if part) + return output[-2000:] or f"Command exited with code {result.returncode}" + + @staticmethod + def _find_sample_binary(search_root: Path, sample_name: str) -> Optional[Path]: + for name in (sample_name, sample_name.replace("_", "")): + direct = search_root / name + if direct.is_file() and os.access(direct, os.X_OK): + return direct + matches = sorted( + path + for path in search_root.rglob(name) + if path.is_file() and os.access(path, os.X_OK) + ) + if matches: + return matches[0] + return None + + def _run_sample( + self, binary: Path, timeout: int, env: Optional[Dict[str, str]] = None + ) -> Tuple[str, str]: + """Run a compiled CUDA sample.""" + try: + result = subprocess.run( + [str(binary)], + cwd=str(binary.parent), + capture_output=True, + text=True, + errors="replace", + env=env, + timeout=timeout, + ) + return self._classify_run_result(result) + except subprocess.TimeoutExpired: + return "fail", f"Execution timed out after {timeout} seconds" + except FileNotFoundError as exc: + return "fail", str(exc) + + @staticmethod + def _classify_run_result( + result: subprocess.CompletedProcess, + ) -> Tuple[str, str]: + output = "\n".join(part for part in (result.stdout, result.stderr) if part) + normalized = output.lower() + if result.returncode == 2 or any( + marker in normalized + for marker in ("sample waived", "waiving sample", "result = waived") + ): + return "skip", output[-2000:] or "Sample waived" + if result.returncode == 0 and not re.search(r"result\s*=\s*fail", normalized): + return "pass", "" + return ( + "fail", + output[-2000:] or f"Executable exited with code {result.returncode}", + ) + + def _clean_make_sample( + self, sample_dir: Path, env: Dict[str, str], timeout: int + ) -> None: + try: + self._run_build_command(["make", "clean"], sample_dir, env, timeout) + except (FileNotFoundError, subprocess.TimeoutExpired): + logger.warning("Failed to clean CUDA sample build: %s", sample_dir) diff --git a/infinibench/dispatcher.py b/infinibench/dispatcher.py index a6870ef2..fb03eed9 100644 --- a/infinibench/dispatcher.py +++ b/infinibench/dispatcher.py @@ -24,6 +24,10 @@ (TestCategory.INFER, "vllm"): lambda: _create_inference_adapter(), (TestCategory.TRAIN, "megatron"): lambda: _create_training_adapter(), (TestCategory.TRAIN, "infinitrain"): lambda: _create_training_adapter(), + ( + TestCategory.COMPATIBILITY, + "cudasamples", + ): lambda: _create_compatibility_adapter(), } @@ -69,6 +73,13 @@ def _create_training_adapter(): return TrainingAdapter() +def _create_compatibility_adapter(): + """Create compatibility adapter (lazy import).""" + from infinibench.compatibility.compatibility_adapter import CompatibilityAdapter + + return CompatibilityAdapter() + + class Dispatcher: """Test orchestration dispatcher for managing test executions.""" diff --git a/infinibench/hardware/constants.py b/infinibench/hardware/constants.py index 03c6fb86..ec335621 100644 --- a/infinibench/hardware/constants.py +++ b/infinibench/hardware/constants.py @@ -23,41 +23,81 @@ "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "cuda", "cache_parser": "cuda", + "compilers": ("nvcc", "/usr/local/cuda/bin/nvcc"), + "detection_tools": ("nvcc", "nvidia-smi"), }, "metax": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "metax", "cache_parser": "cuda", + "compilers": ( + "/opt/maca/tools/cu-bridge/bin/cucc", + "cucc", + "/opt/maca/mxgpu_llvm/bin/mxcc", + "mxcc", + ), + "detection_tools": ( + "/opt/maca/tools/cu-bridge/bin/cucc", + "cucc", + "mxcc", + ), }, "corex": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "corex", "cache_parser": "cuda", + "compilers": ( + "/usr/local/corex/bin/clang++", + "/usr/local/corex/bin/nvcc", + "nvcc", + ), + "detection_tools": ( + "/usr/local/corex/bin/ixsmi", + "/usr/local/corex/bin/clang++", + ), }, "hygon": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "hygon", "cache_parser": "cuda", + "compilers": ("/opt/dtk/bin/hipcc", "hipcc"), + "detection_tools": ("/opt/dtk/bin/hy-smi", "hy-smi"), + "conditional_detection_tools": (("/opt/dtk", "hipcc"),), }, "moore": { "binary_name": "cuda_perf_suite", "benchmark_subdir": "cuda-memory-benchmark", "build_platform": "moore", "cache_parser": "cuda", + "compilers": ("/usr/local/musa/bin/mcc", "mcc"), + "detection_tools": ("mcc", "mthreads-gmi"), }, "cambricon": { "binary_name": "mlu_perf_suite", "benchmark_subdir": "cambricon-memory-benchmark", "build_platform": None, "cache_parser": "cambricon", + "detection_tools": ("cncc",), + "detection_paths": ("/usr/local/neuware",), }, "ascend": { "binary_name": "npu_perf_suite", "benchmark_subdir": "ascend-memory-benchmark", "build_platform": None, "cache_parser": "ascend", + "detection_tools": ("npu-smi", "atc"), + "detection_paths": ("/usr/local/Ascend/ascend-toolkit",), }, } + +PLATFORM_DETECTION_ORDER = ( + "ascend", + "cambricon", + "moore", + "metax", + "hygon", + "corex", +) diff --git a/infinibench/hardware/hardware_adapter.py b/infinibench/hardware/hardware_adapter.py index 1b8c4dfb..a5a1e34a 100644 --- a/infinibench/hardware/hardware_adapter.py +++ b/infinibench/hardware/hardware_adapter.py @@ -25,7 +25,11 @@ InfiniBenchJson, ) from infinibench.common.csv_utils import create_timeseries_metric -from infinibench.hardware.constants import PLATFORM_ALIASES, PLATFORM_CONFIGS +from infinibench.hardware.constants import ( + PLATFORM_ALIASES, + PLATFORM_CONFIGS, + PLATFORM_DETECTION_ORDER, +) from infinibench.utils.time_utils import get_timestamp logger = logging.getLogger(__name__) @@ -33,39 +37,34 @@ def detect_platform() -> str: """Detect the installed accelerator toolchain.""" - if ( - shutil.which("npu-smi") - or shutil.which("atc") - or Path("/usr/local/Ascend/ascend-toolkit").exists() - ): - return "ascend" - if shutil.which("cncc") or Path("/usr/local/neuware").exists(): - return "cambricon" - if shutil.which("mcc") or shutil.which("mthreads-gmi"): - return "moore" - - maca_path = Path("/opt/maca") - if ( - (maca_path / "tools" / "cu-bridge" / "bin" / "cucc").exists() - or shutil.which("cucc") - or shutil.which("mxcc") - ): - return "metax" + for platform in PLATFORM_DETECTION_ORDER: + platform_config = PLATFORM_CONFIGS[platform] + tools = platform_config["detection_tools"] + if any(_tool_exists(tool) for tool in tools): + return platform + paths = platform_config.get("detection_paths", ()) + if any(_path_exists(path) for path in paths): + return platform + conditional_tools = platform_config.get("conditional_detection_tools", ()) + if any( + _path_exists(root) and _tool_exists(tool) + for root, tool in conditional_tools + ): + return platform + return "cuda" - dtk_path = Path("/opt/dtk") - if ( - (dtk_path / "bin" / "hy-smi").exists() - or shutil.which("hy-smi") - or (dtk_path.exists() and shutil.which("hipcc")) - ): - return "hygon" - corex_path = Path("/usr/local/corex") - if (corex_path / "bin" / "ixsmi").exists() or ( - corex_path / "bin" / "clang++" - ).exists(): - return "corex" - return "cuda" +def _tool_exists(tool: str) -> bool: + """Return whether a command or absolute tool path is available.""" + path = Path(tool) + if path.is_absolute(): + return path.is_file() + return shutil.which(tool) is not None + + +def _path_exists(path: str) -> bool: + """Return whether a platform-specific installation path exists.""" + return Path(path).exists() class HardwareTestAdapter(BaseAdapter): diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py new file mode 100644 index 00000000..10364278 --- /dev/null +++ b/tests/test_compatibility_adapter.py @@ -0,0 +1,329 @@ +import os +import subprocess +from pathlib import Path + +import pytest + +from infinibench.common.constants import CUDA_SAMPLE_CONFIGS +from infinibench.compatibility.compatibility_adapter import CompatibilityAdapter +from infinibench.dispatcher import Dispatcher + + +def _sample(root: Path, category: str, name: str, manifest: str) -> Path: + sample_dir = root / "Samples" / category / name + sample_dir.mkdir(parents=True) + content = f"project({name})\n" if manifest == "CMakeLists.txt" else "# test\n" + (sample_dir / manifest).write_text(content, encoding="utf-8") + return sample_dir + + +@pytest.fixture +def compiler_available(monkeypatch): + monkeypatch.setattr( + "infinibench.compatibility.compatibility_adapter.shutil.which", + lambda candidate, **_: candidate, + ) + + +def test_discovers_nested_cmake_and_make_samples(tmp_path): + (tmp_path / "Samples").mkdir() + (tmp_path / "Samples" / "CMakeLists.txt").write_text("# aggregate\n") + category = tmp_path / "Samples" / "0_Introduction" + category.mkdir() + (category / "CMakeLists.txt").write_text("# aggregate\n") + vector_add = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + clock = _sample(tmp_path, "0_Introduction", "clock", "Makefile") + + discovered = CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", None + ) + + assert discovered == [clock, vector_add] + + +def test_sample_filter_rejects_unknown_names(tmp_path): + _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + + with pytest.raises(ValueError, match="missingSample"): + CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", ["vectorAdd", "missingSample"] + ) + + +def test_discovery_excludes_nested_cmake_group_manifests(tmp_path): + group = tmp_path / "Samples" / "8_Platform_Specific" / "Tegra" + group.mkdir(parents=True) + (group / "CMakeLists.txt").write_text( + "add_subdirectory(simpleGLES)\n", encoding="utf-8" + ) + sample = _sample( + tmp_path, + "8_Platform_Specific/Tegra", + "simpleGLES", + "CMakeLists.txt", + ) + + discovered = CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", None + ) + + assert discovered == [sample] + + +def test_sample_filter_rejects_an_empty_list(tmp_path): + _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + + with pytest.raises(ValueError, match="non-empty"): + CompatibilityAdapter()._discover_sample_dirs(tmp_path / "Samples", []) + + +@pytest.mark.parametrize("sms", ["", '80")\nmessage(FATAL_ERROR injected)']) +def test_architecture_rejects_empty_or_cmake_control_characters(sms): + with pytest.raises(ValueError, match="invalid architecture"): + CompatibilityAdapter._validate_architectures(sms) + + +@pytest.mark.parametrize( + ("build_system", "manifest", "sms", "compiler", "jobs"), + [ + ("cmake", "CMakeLists.txt", "80", "/usr/local/cuda/bin/nvcc", 7), + ("make", "Makefile", "70", "/usr/local/musa/bin/mcc", 3), + ], +) +def test_build_commands_include_compiler_arch_and_jobs( + tmp_path, monkeypatch, build_system, manifest, sms, compiler, jobs +): + sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", manifest) + build_dir = tmp_path / "build" + expected_binary = ( + build_dir / "build" if build_system == "cmake" else sample_dir + ) / "vectorAdd" + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + is_build = command[:2] == ["cmake", "--build"] or ( + command[0] == "make" and "clean" not in command + ) + if is_build: + expected_binary.parent.mkdir(parents=True, exist_ok=True) + expected_binary.write_text("binary", encoding="utf-8") + expected_binary.chmod(0o755) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + binary = CompatibilityAdapter()._compile_sample( + sample_dir, build_dir, {"SMS": sms, "CUDACXX": compiler}, 30, jobs, build_system + ) + + expected = ( + [ + "cmake", + "--build", + str(build_dir / "build"), + "--parallel", + str(jobs), + ] + if build_system == "cmake" + else ["make", f"-j{jobs}", f"SMS={sms}", f"NVCC={compiler}"] + ) + assert binary == expected_binary + assert expected in calls + + +def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): + sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + wrapper_dir = tmp_path / "wrapper" + + CompatibilityAdapter._write_cmake_wrapper(wrapper_dir, sample_dir, "ivcore20") + + wrapper = (wrapper_dir / "CMakeLists.txt").read_text(encoding="utf-8") + assert f'add_subdirectory("{sample_dir.resolve().as_posix()}" sample)' in wrapper + assert 'PROPERTY CUDA_ARCHITECTURES "ivcore20"' in wrapper + + +@pytest.mark.parametrize("platform", ["metax", "corex"]) +def test_vendor_make_args_remove_nvidia_only_flags(platform): + args = " ".join(CUDA_SAMPLE_CONFIGS[platform]["make_args"]) + + assert all(flag not in args for flag in ("--threads", "-gencode", "-m64")) + + +def test_corex_make_args_keep_source_language_out_of_link_step(): + compile_args, link_args, _ = CUDA_SAMPLE_CONFIGS["corex"]["make_args"] + + assert "-x ivcore" in compile_args + assert "-x ivcore" not in link_args + + +def test_compile_env_matches_selected_vendor_toolkit(monkeypatch, compiler_available): + compiler = "/usr/local/corex/bin/clang++" + monkeypatch.setenv("CUDA_HOME", "/usr/local/cuda") + monkeypatch.setenv("CUDA_PATH", "/usr/local/cuda") + env = CompatibilityAdapter()._build_compile_env("corex", "ivcore20", compiler) + + expected_root = str(Path(compiler).resolve().parent.parent) + assert env["CUDACXX"] == compiler + assert env["CUDA_HOME"] == expected_root + assert env["CUDA_PATH"] == expected_root + + +@pytest.mark.parametrize( + ("compiler", "root_parent_index"), + [ + ("/opt/maca/tools/cu-bridge/bin/cucc", 3), + ("/opt/maca/mxgpu_llvm/bin/mxcc", 2), + ], +) +def test_metax_compile_env_infers_maca_path( + monkeypatch, compiler_available, compiler, root_parent_index +): + monkeypatch.delenv("MACA_PATH", raising=False) + + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + assert env["MACA_PATH"] == str(Path(compiler).resolve().parents[root_parent_index]) + + +def test_metax_compile_env_preserves_explicit_maca_path( + monkeypatch, compiler_available +): + compiler = "/opt/maca/tools/cu-bridge/bin/cucc" + monkeypatch.setenv("MACA_PATH", "/custom/maca") + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + assert env["MACA_PATH"] == "/custom/maca" + + +def test_cuda_sample_metrics_keep_skips_separate(tmp_path, monkeypatch): + samples = [ + _sample(tmp_path, "0_Introduction", name, "CMakeLists.txt") + for name in ("passes", "fails", "waived") + ] + binaries = {sample.name: tmp_path / f"{sample.name}.bin" for sample in samples} + adapter = CompatibilityAdapter() + + monkeypatch.setattr(adapter, "_discover_sample_dirs", lambda *_: samples) + monkeypatch.setattr( + adapter, + "_build_compile_env", + lambda *_: {"SMS": "80", "CUDACXX": "/usr/local/cuda/bin/nvcc"}, + ) + monkeypatch.setattr( + adapter, + "_compile_sample", + lambda sample_dir, *_: binaries[sample_dir.name], + ) + outcomes = { + "passes.bin": ("pass", ""), + "fails.bin": ("fail", "kernel failed"), + "waived.bin": ("skip", "sample waived"), + } + monkeypatch.setattr( + adapter, "_run_sample", lambda binary, *_: outcomes[binary.name] + ) + + config = {"platform": "nvidia", "cuda_samples_dir": str(tmp_path)} + metrics = adapter._run_cuda_samples_test(config) + values = {metric["name"]: metric["value"] for metric in metrics} + + assert config["compiler"] == "/usr/local/cuda/bin/nvcc" + assert config["sms"] == "80" + assert values["compatibility.cuda_samples.run_passed"] == 1 + assert values["compatibility.cuda_samples.run_failed"] == 1 + assert values["compatibility.cuda_samples.run_skipped"] == 1 + assert values["compatibility.cuda_samples.run_pass_rate"] == 33.33 + + +@pytest.mark.parametrize("failure_stage", ["compile", "run"]) +def test_sample_failures_are_accounted_for(tmp_path, monkeypatch, failure_stage): + sample = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + adapter = CompatibilityAdapter() + monkeypatch.setattr(adapter, "_discover_sample_dirs", lambda *_: [sample]) + monkeypatch.setattr( + adapter, + "_build_compile_env", + lambda *_: {"SMS": "80", "CUDACXX": "/usr/local/cuda/bin/nvcc"}, + ) + + def compile_sample(*_): + if failure_stage == "compile": + raise RuntimeError("compiler failed") + return tmp_path / "vectorAdd" + + def run_sample(*_): + raise RuntimeError("runtime failed") + + monkeypatch.setattr(adapter, "_compile_sample", compile_sample) + monkeypatch.setattr(adapter, "_run_sample", run_sample) + + metrics = adapter._run_cuda_samples_test( + {"platform": "nvidia", "cuda_samples_dir": str(tmp_path)} + ) + values = {metric["name"]: metric["value"] for metric in metrics} + details = values["compatibility.cuda_samples.details"] + + compile_passed = int(failure_stage == "run") + assert values["compatibility.cuda_samples.compile_passed"] == compile_passed + assert values["compatibility.cuda_samples.run_failed"] == compile_passed + assert values["compatibility.cuda_samples.run_skipped"] == 0 + assert details[0]["compile_result"] == ("pass" if compile_passed else "fail") + assert details[0]["run_result"] == ("fail" if compile_passed else "not_run") + + +def test_run_sample_classifies_cuda_waiver(tmp_path): + binary = tmp_path / "sample" + binary.write_text("binary", encoding="utf-8") + binary.chmod(0o755) + adapter = CompatibilityAdapter() + + result, error = adapter._classify_run_result( + subprocess.CompletedProcess([str(binary)], 2, "Sample waived", "") + ) + + assert result == "skip" + assert "waived" in error.lower() + + +def test_run_sample_uses_the_binary_directory(tmp_path, monkeypatch): + binary = tmp_path / "build" / "sample" + binary.parent.mkdir() + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, "Result = PASS", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result, _ = CompatibilityAdapter()._run_sample(binary, 30) + + assert result == "pass" + assert calls == [ + ( + [str(binary)], + { + "cwd": str(binary.parent), + "capture_output": True, + "text": True, + "errors": "replace", + "env": None, + "timeout": 30, + }, + ) + ] + + +def test_metrics_reject_an_empty_sample_set(): + with pytest.raises(ValueError, match="No CUDA samples selected"): + CompatibilityAdapter._build_metrics([]) + + +def test_dispatcher_only_registers_cuda_samples_compatibility(): + assert isinstance( + Dispatcher()._create_adapter("compatibility", "cudasamples"), + CompatibilityAdapter, + ) + for framework in ("megatron", "vllm", "infinilm"): + with pytest.raises(ValueError, match="Adapter not registered"): + Dispatcher()._create_adapter("compatibility", framework) diff --git a/tests/test_hardware_adapter.py b/tests/test_hardware_adapter.py index 585d9afd..e373a2cc 100644 --- a/tests/test_hardware_adapter.py +++ b/tests/test_hardware_adapter.py @@ -4,7 +4,7 @@ from infinibench.dispatcher import Dispatcher from infinibench.hardware import hardware_adapter -from infinibench.hardware.hardware_adapter import HardwareTestAdapter +from infinibench.hardware.hardware_adapter import HardwareTestAdapter, detect_platform CUDA_OUTPUT = """ Direction: Host to Device @@ -178,6 +178,54 @@ def test_runtime_detection_ignores_testcase_framework(tmp_path, monkeypatch): assert adapter._get_device_type({}) == "moore" +def test_runtime_detection_uses_shared_detection_tools(monkeypatch): + monkeypatch.setattr( + hardware_adapter, + "_tool_exists", + lambda tool: tool == "/opt/maca/tools/cu-bridge/bin/cucc", + ) + + assert detect_platform() == "metax" + + +def test_runtime_detection_accepts_ascend_toolkit_path(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda *_: False) + monkeypatch.setattr( + hardware_adapter, + "_path_exists", + lambda path: path == "/usr/local/Ascend/ascend-toolkit", + ) + + assert detect_platform() == "ascend" + + +def test_runtime_detection_accepts_cambricon_toolkit_path(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda *_: False) + monkeypatch.setattr( + hardware_adapter, + "_path_exists", + lambda path: path == "/usr/local/neuware", + ) + + assert detect_platform() == "cambricon" + + +def test_runtime_detection_does_not_treat_rocm_hipcc_as_hygon(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda tool: tool == "hipcc") + monkeypatch.setattr(hardware_adapter, "_path_exists", lambda *_: False) + + assert detect_platform() == "cuda" + + +def test_runtime_detection_accepts_hipcc_inside_dtk(monkeypatch): + monkeypatch.setattr(hardware_adapter, "_tool_exists", lambda tool: tool == "hipcc") + monkeypatch.setattr( + hardware_adapter, "_path_exists", lambda path: path == "/opt/dtk" + ) + + assert detect_platform() == "hygon" + + @pytest.mark.parametrize("device", ["cuda", "metax", "corex", "hygon", "moore"]) def test_cuda_compatible_platforms_share_binary(tmp_path, device): cuda_binary = tmp_path / "cuda_perf_suite" diff --git a/tests/test_hardware_detection.py b/tests/test_hardware_detection.py index 3878a8b4..1ef5c982 100644 --- a/tests/test_hardware_detection.py +++ b/tests/test_hardware_detection.py @@ -5,6 +5,7 @@ from infinibench.utils import hardware_detector from infinibench.utils.hardware_detector import HardwareDetector + MTHREADS_OUTPUT = """ Attached GPUs : 2 From bd7a7c60ec7fe08b1fc524042c7ebece32a6d89c Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 11:09:29 +0800 Subject: [PATCH 2/7] fix: target CoreX BI-V150 with ivcore11 --- docs/compatibility.md | 5 ++++- infinibench/common/constants.py | 6 +++--- .../hardware/cuda-memory-benchmark/CMakeLists.txt | 2 +- .../hardware/cuda-memory-benchmark/build.sh | 2 +- tests/test_compatibility_adapter.py | 15 ++++++++++++--- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index d9214458..fa12f1f3 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -49,11 +49,14 @@ values and Make arguments live in `infinibench.common.constants`. | --- | --- | --- | | NVIDIA | `nvcc` | `80` | | MetaX | `cucc` (falls back to `mxcc`) | `70` | -| Iluvatar CoreX | `/usr/local/corex/bin/clang++` | `ivcore20` | +| Iluvatar CoreX (BI-V150/TG150) | `/usr/local/corex/bin/clang++` | `ivcore11` | The supported canonical platform names are `cuda`, `metax`, and `corex`. The existing aliases `nvidia` and `iluvatar` are also accepted. +The CoreX default targets the BI-V150/TG150 validated on `tianshu58`. Override +`sms` and `make_args` together when testing a different Iluvatar architecture. + Set `compiler`, `sms`, or `make_args` in the input when the installed vendor SDK uses a wrapper or different target. Arguments are passed directly as an argument list; shell expansion is not performed. The resolved default compiler diff --git a/infinibench/common/constants.py b/infinibench/common/constants.py index 5bcdaab1..0f62d1bd 100644 --- a/infinibench/common/constants.py +++ b/infinibench/common/constants.py @@ -254,11 +254,11 @@ class InfiniCoreResult: ), }, "corex": { - "sms": "ivcore20", + "sms": "ivcore11", "make_args": ( - "ALL_CCFLAGS=-x ivcore --cuda-gpu-arch=ivcore20 " + "ALL_CCFLAGS=-x ivcore --cuda-gpu-arch=ivcore11 " "--cuda-path=/usr/local/corex --std=c++11", - "ALL_LDFLAGS=--cuda-gpu-arch=ivcore20 " + "ALL_LDFLAGS=--cuda-gpu-arch=ivcore11 " "--cuda-path=/usr/local/corex -L/usr/local/corex/lib " "-Wl,-rpath,/usr/local/corex/lib -lcudart", "GENCODE_FLAGS=", diff --git a/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt b/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt index 223a6627..68bb8213 100644 --- a/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt +++ b/infinibench/hardware/cuda-memory-benchmark/CMakeLists.txt @@ -63,7 +63,7 @@ elseif(PLATFORM STREQUAL "corex") # CoreX adapted CMake handles CUDA compiler automatically (clang++) set(CMAKE_CUDA_STANDARD 17) set(CMAKE_CUDA_STANDARD_REQUIRED ON) - set(CMAKE_CUDA_ARCHITECTURES "ivcore20" CACHE STRING "CoreX GPU architectures") + set(CMAKE_CUDA_ARCHITECTURES "ivcore11" CACHE STRING "CoreX GPU architectures") include_directories($ENV{COREX_PATH}/include) link_directories($ENV{COREX_PATH}/lib64) diff --git a/infinibench/hardware/cuda-memory-benchmark/build.sh b/infinibench/hardware/cuda-memory-benchmark/build.sh index 723f3551..f6aaf1f5 100644 --- a/infinibench/hardware/cuda-memory-benchmark/build.sh +++ b/infinibench/hardware/cuda-memory-benchmark/build.sh @@ -110,7 +110,7 @@ elif [[ "$PLATFORM" == "corex" ]]; then echo -e "${YELLOW}Configuring with CoreX CMake...${NC}" cmake .. -DCMAKE_BUILD_TYPE=Release -DPLATFORM=corex \ - -DCMAKE_CUDA_ARCHITECTURES=ivcore20 + -DCMAKE_CUDA_ARCHITECTURES=ivcore11 echo -e "${YELLOW}Building...${NC}" make -j$(nproc) diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index 10364278..7821d731 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -135,11 +135,11 @@ def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") wrapper_dir = tmp_path / "wrapper" - CompatibilityAdapter._write_cmake_wrapper(wrapper_dir, sample_dir, "ivcore20") + CompatibilityAdapter._write_cmake_wrapper(wrapper_dir, sample_dir, "ivcore11") wrapper = (wrapper_dir / "CMakeLists.txt").read_text(encoding="utf-8") assert f'add_subdirectory("{sample_dir.resolve().as_posix()}" sample)' in wrapper - assert 'PROPERTY CUDA_ARCHITECTURES "ivcore20"' in wrapper + assert 'PROPERTY CUDA_ARCHITECTURES "ivcore11"' in wrapper @pytest.mark.parametrize("platform", ["metax", "corex"]) @@ -156,11 +156,20 @@ def test_corex_make_args_keep_source_language_out_of_link_step(): assert "-x ivcore" not in link_args +def test_corex_defaults_target_bi_v150(): + config = CUDA_SAMPLE_CONFIGS["corex"] + compile_args, link_args, _ = config["make_args"] + + assert config["sms"] == "ivcore11" + assert "--cuda-gpu-arch=ivcore11" in compile_args + assert "--cuda-gpu-arch=ivcore11" in link_args + + def test_compile_env_matches_selected_vendor_toolkit(monkeypatch, compiler_available): compiler = "/usr/local/corex/bin/clang++" monkeypatch.setenv("CUDA_HOME", "/usr/local/cuda") monkeypatch.setenv("CUDA_PATH", "/usr/local/cuda") - env = CompatibilityAdapter()._build_compile_env("corex", "ivcore20", compiler) + env = CompatibilityAdapter()._build_compile_env("corex", "ivcore11", compiler) expected_root = str(Path(compiler).resolve().parent.parent) assert env["CUDACXX"] == compiler From 3fa9a4f4c8f4bc03ec6791d7c186e9b719ca6c6d Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 14:49:31 +0800 Subject: [PATCH 3/7] fix: resolve relative CUDA sample binaries --- .../compatibility/compatibility_adapter.py | 1 + tests/test_compatibility_adapter.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py index 5440db52..973ac66d 100644 --- a/infinibench/compatibility/compatibility_adapter.py +++ b/infinibench/compatibility/compatibility_adapter.py @@ -467,6 +467,7 @@ def _run_sample( self, binary: Path, timeout: int, env: Optional[Dict[str, str]] = None ) -> Tuple[str, str]: """Run a compiled CUDA sample.""" + binary = binary.resolve() try: result = subprocess.run( [str(binary)], diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index 7821d731..e272990e 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -323,6 +323,24 @@ def fake_run(command, **kwargs): ] +def test_run_sample_resolves_a_relative_binary_path(tmp_path, monkeypatch): + binary = Path("build") / "sample" + monkeypatch.chdir(tmp_path) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs["cwd"])) + return subprocess.CompletedProcess(command, 0, "Result = PASS", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result, _ = CompatibilityAdapter()._run_sample(binary, 30) + + resolved_binary = binary.resolve() + assert result == "pass" + assert calls == [([str(resolved_binary)], str(resolved_binary.parent))] + + def test_metrics_reject_an_empty_sample_set(): with pytest.raises(ValueError, match="No CUDA samples selected"): CompatibilityAdapter._build_metrics([]) From 3f592e3d585c08caeffa7f15390ae1d0e8c5c52f Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 15:37:27 +0800 Subject: [PATCH 4/7] build: add CUDA Samples submodule --- .flake8 | 1 + .gitmodules | 3 ++ docs/compatibility.md | 29 +++++++++++-------- .../compatibility/compatibility_adapter.py | 8 ++--- pyproject.toml | 1 + submodules/cuda-samples | 1 + tests/test_compatibility_adapter.py | 11 ++++++- 7 files changed, 36 insertions(+), 18 deletions(-) create mode 160000 submodules/cuda-samples diff --git a/.flake8 b/.flake8 index d41ad4cd..9d6de545 100644 --- a/.flake8 +++ b/.flake8 @@ -25,6 +25,7 @@ exclude = venv, build, dist, + submodules, *.egg-info # Show source code for each error diff --git a/.gitmodules b/.gitmodules index 8662647b..3cef7753 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "submodules/nccl-tests"] path = submodules/nccl-tests url = git@github.com:NVIDIA/nccl-tests.git +[submodule "submodules/cuda-samples"] + path = submodules/cuda-samples + url = https://github.com/spike-zhu/cuda-samples.git diff --git a/docs/compatibility.md b/docs/compatibility.md index fa12f1f3..38dcca4c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -13,8 +13,6 @@ adapter execution error. "testcase": "compatibility.CudaSamples.PassRate", "config": { "platform": "nvidia", - "cuda_samples_dir": "/workspace/cuda-samples", - "build_system": "make", "sample_filter": ["vectorAdd", "matrixMul", "clock"], "timeout_per_sample": 180, "jobs": 4 @@ -22,15 +20,22 @@ adapter execution error. } ``` -`cuda_samples_dir` must contain a `Samples` directory. The adapter recursively -discovers `Samples//` directories containing a Makefile or a -standalone CMake project. CMake grouping manifests that only aggregate child -directories are excluded. A requested sample name that is not found, or an -empty `sample_filter`, is a configuration error instead of a successful -zero-sample result. +Initialize the bundled CUDA Samples revision before running compatibility tests: -The InfiniPerf cuda-samples submodule pins the CMake-based `master` revision. -Its `batch_test` branch provides the Makefiles used by the original InfiniPerf +```bash +git submodule update --init submodules/cuda-samples +``` + +The adapter uses `submodules/cuda-samples` by default. An optional +`cuda_samples_dir` override must contain a `Samples` directory. The adapter +recursively discovers `Samples//` directories containing a +Makefile or a standalone CMake project. CMake grouping manifests that only +aggregate child directories are excluded. A requested sample name that is not +found, or an empty `sample_filter`, is a configuration error instead of a +successful zero-sample result. + +The bundled submodule pins the CMake-based `master` revision at `7b601789`. +The upstream `batch_test` branch provides the Makefiles used by the original compatibility workflow. The adapter supports both layouts; use `build_system` to select `cmake`, `make`, or the default `auto` detection. @@ -54,8 +59,8 @@ values and Make arguments live in `infinibench.common.constants`. The supported canonical platform names are `cuda`, `metax`, and `corex`. The existing aliases `nvidia` and `iluvatar` are also accepted. -The CoreX default targets the BI-V150/TG150 validated on `tianshu58`. Override -`sms` and `make_args` together when testing a different Iluvatar architecture. +The CoreX default targets BI-V150/TG150. Override `sms` and `make_args` +together when testing a different Iluvatar architecture. Set `compiler`, `sms`, or `make_args` in the input when the installed vendor SDK uses a wrapper or different target. Arguments are passed directly as an diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py index 973ac66d..6cca6fac 100644 --- a/infinibench/compatibility/compatibility_adapter.py +++ b/infinibench/compatibility/compatibility_adapter.py @@ -25,11 +25,9 @@ logger = logging.getLogger(__name__) _METRIC_PREFIX = "compatibility.cuda_samples" -# Repo root for finding cuda-samples -_REPO_ROOT = Path(__file__).resolve().parents[3] -_CUDA_SAMPLES_DIR = ( - _REPO_ROOT / "InfiniPerf" / "benchmarks" / "compatibility" / "cuda-samples" -) +# Repository submodule containing the pinned CUDA Samples revision. +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CUDA_SAMPLES_DIR = _REPO_ROOT / "submodules" / "cuda-samples" class CompatibilityAdapter(BaseAdapter): diff --git a/pyproject.toml b/pyproject.toml index a96214a2..d7996997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ exclude = ''' | buck-out | build | dist + | submodules | __pycache__ )/ ''' diff --git a/submodules/cuda-samples b/submodules/cuda-samples new file mode 160000 index 00000000..7b601789 --- /dev/null +++ b/submodules/cuda-samples @@ -0,0 +1 @@ +Subproject commit 7b60178984e96bc09d066077d5455df71fee2a9f diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index e272990e..8174ad4f 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -5,7 +5,10 @@ import pytest from infinibench.common.constants import CUDA_SAMPLE_CONFIGS -from infinibench.compatibility.compatibility_adapter import CompatibilityAdapter +from infinibench.compatibility.compatibility_adapter import ( + _CUDA_SAMPLES_DIR, + CompatibilityAdapter, +) from infinibench.dispatcher import Dispatcher @@ -25,6 +28,12 @@ def compiler_available(monkeypatch): ) +def test_default_cuda_samples_dir_uses_repository_submodule(): + repository_root = Path(__file__).resolve().parents[1] + + assert _CUDA_SAMPLES_DIR == repository_root / "submodules" / "cuda-samples" + + def test_discovers_nested_cmake_and_make_samples(tmp_path): (tmp_path / "Samples").mkdir() (tmp_path / "Samples" / "CMakeLists.txt").write_text("# aggregate\n") From ce0439a352e41bff7c23ebc5026af29364b45f12 Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 16:18:42 +0800 Subject: [PATCH 5/7] fix: use MetaX CMake wrapper for CUDA Samples --- docs/compatibility.md | 4 +- infinibench/common/constants.py | 6 +++ .../compatibility/compatibility_adapter.py | 22 ++++++++- tests/test_compatibility_adapter.py | 48 ++++++++++++++++--- 4 files changed, 69 insertions(+), 11 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 38dcca4c..465ab506 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -67,8 +67,8 @@ SDK uses a wrapper or different target. Arguments are passed directly as an argument list; shell expansion is not performed. The resolved default compiler and architecture are added to the result config when they were not explicit in the input. For MetaX, the adapter also infers `MACA_PATH` from the resolved -`cucc` or `mxcc` location when the variable is unset. An explicit `MACA_PATH` -is preserved. +`cucc` or `mxcc` location when the variable is unset. It uses the cu-bridge +`cmake_maca` wrapper when available. An explicit `MACA_PATH` is preserved. For non-NVIDIA Makefile builds, the default arguments remove NVIDIA-only `--threads`, `-gencode`, and `-m64` flags. Platform support is declared only diff --git a/infinibench/common/constants.py b/infinibench/common/constants.py index 0f62d1bd..64a992da 100644 --- a/infinibench/common/constants.py +++ b/infinibench/common/constants.py @@ -247,6 +247,12 @@ class InfiniCoreResult: }, "metax": { "sms": "70", + "cmake_commands": ( + "/opt/maca/tools/cu-bridge/tools/cmake_maca", + "cmake_maca", + "cmake", + ), + "extra_env": {"CUCC_CMAKE_ENTRY": "2"}, "make_args": ( "ALL_CCFLAGS=--std=c++11", "ALL_LDFLAGS=", diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py index 6cca6fac..b1829bba 100644 --- a/infinibench/compatibility/compatibility_adapter.py +++ b/infinibench/compatibility/compatibility_adapter.py @@ -323,6 +323,23 @@ def _build_compile_env( if maca_path: env["MACA_PATH"] = maca_path + sample_config = CUDA_SAMPLE_CONFIGS[platform] + for name, value in sample_config.get("extra_env", {}).items(): + env.setdefault(name, value) + cmake_candidates = sample_config.get("cmake_commands", ("cmake",)) + cmake_command = next( + ( + resolved + for candidate in cmake_candidates + if (resolved := shutil.which(candidate, path=env.get("PATH"))) + ), + None, + ) + if cmake_command: + env["CMAKE_COMMAND"] = cmake_command + command_dir = str(Path(cmake_command).resolve().parent) + env["PATH"] = os.pathsep.join((command_dir, env.get("PATH", ""))) + return env @staticmethod @@ -347,13 +364,14 @@ def _compile_sample( """Compile one CUDA sample and return its executable.""" sms = env.get("SMS", "80") if build_system == "cmake": + cmake_command = env.get("CMAKE_COMMAND", "cmake") wrapper_dir = build_dir / "source" cmake_build_dir = build_dir / "build" self._write_cmake_wrapper(wrapper_dir, sample_dir, sms) cmake_build_dir.mkdir(parents=True, exist_ok=True) configure_result = self._run_build_command( [ - "cmake", + cmake_command, "-S", str(wrapper_dir), "-B", @@ -368,7 +386,7 @@ def _compile_sample( if configure_result.returncode: raise RuntimeError(self._command_error(configure_result)) command = [ - "cmake", + cmake_command, "--build", str(cmake_build_dir), "--parallel", diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index 8174ad4f..1aadaf95 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -93,14 +93,28 @@ def test_architecture_rejects_empty_or_cmake_control_characters(sms): @pytest.mark.parametrize( - ("build_system", "manifest", "sms", "compiler", "jobs"), + ("build_system", "manifest", "sms", "compiler", "cmake_command", "jobs"), [ - ("cmake", "CMakeLists.txt", "80", "/usr/local/cuda/bin/nvcc", 7), - ("make", "Makefile", "70", "/usr/local/musa/bin/mcc", 3), + ( + "cmake", + "CMakeLists.txt", + "80", + "/usr/local/cuda/bin/nvcc", + "/vendor/bin/cmake_maca", + 7, + ), + ("make", "Makefile", "70", "/usr/local/musa/bin/mcc", "cmake", 3), ], ) def test_build_commands_include_compiler_arch_and_jobs( - tmp_path, monkeypatch, build_system, manifest, sms, compiler, jobs + tmp_path, + monkeypatch, + build_system, + manifest, + sms, + compiler, + cmake_command, + jobs, ): sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", manifest) build_dir = tmp_path / "build" @@ -111,7 +125,7 @@ def test_build_commands_include_compiler_arch_and_jobs( def fake_run(command, **kwargs): calls.append(command) - is_build = command[:2] == ["cmake", "--build"] or ( + is_build = (len(command) > 1 and command[1] == "--build") or ( command[0] == "make" and "clean" not in command ) if is_build: @@ -122,12 +136,21 @@ def fake_run(command, **kwargs): monkeypatch.setattr(subprocess, "run", fake_run) binary = CompatibilityAdapter()._compile_sample( - sample_dir, build_dir, {"SMS": sms, "CUDACXX": compiler}, 30, jobs, build_system + sample_dir, + build_dir, + { + "SMS": sms, + "CUDACXX": compiler, + "CMAKE_COMMAND": cmake_command, + }, + 30, + jobs, + build_system, ) expected = ( [ - "cmake", + cmake_command, "--build", str(build_dir / "build"), "--parallel", @@ -213,6 +236,17 @@ def test_metax_compile_env_preserves_explicit_maca_path( assert env["MACA_PATH"] == "/custom/maca" +def test_metax_compile_env_selects_cu_bridge_cmake(monkeypatch, compiler_available): + compiler = "/opt/maca/tools/cu-bridge/bin/cucc" + + env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) + + cmake_command = "/opt/maca/tools/cu-bridge/tools/cmake_maca" + assert env["CMAKE_COMMAND"] == cmake_command + assert env["CUCC_CMAKE_ENTRY"] == "2" + assert env["PATH"].split(os.pathsep)[0] == str(Path(cmake_command).resolve().parent) + + def test_cuda_sample_metrics_keep_skips_separate(tmp_path, monkeypatch): samples = [ _sample(tmp_path, "0_Introduction", name, "CMakeLists.txt") From 3f7f285924d247d7849647a3b4333557ed912612 Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 16:32:03 +0800 Subject: [PATCH 6/7] fix: let MetaX CMake wrapper manage its toolchain --- .../compatibility/compatibility_adapter.py | 31 ++++++++++----- tests/test_compatibility_adapter.py | 38 +++++++++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/infinibench/compatibility/compatibility_adapter.py b/infinibench/compatibility/compatibility_adapter.py index b1829bba..cf3264f7 100644 --- a/infinibench/compatibility/compatibility_adapter.py +++ b/infinibench/compatibility/compatibility_adapter.py @@ -365,22 +365,32 @@ def _compile_sample( sms = env.get("SMS", "80") if build_system == "cmake": cmake_command = env.get("CMAKE_COMMAND", "cmake") + cmake_env = env.copy() wrapper_dir = build_dir / "source" cmake_build_dir = build_dir / "build" self._write_cmake_wrapper(wrapper_dir, sample_dir, sms) cmake_build_dir.mkdir(parents=True, exist_ok=True) + configure_command = [ + cmake_command, + "-S", + str(wrapper_dir), + "-B", + str(cmake_build_dir), + ] + if Path(cmake_command).name == "cmake_maca": + cmake_env.pop("CUDACXX", None) + cmake_env["WCUDA_HOME"] = str(build_dir / "cmake-maca") + else: + configure_command.extend( + [ + f"-DCMAKE_CUDA_ARCHITECTURES={sms}", + f"-DCMAKE_CUDA_COMPILER={env['CUDACXX']}", + ] + ) configure_result = self._run_build_command( - [ - cmake_command, - "-S", - str(wrapper_dir), - "-B", - str(cmake_build_dir), - f"-DCMAKE_CUDA_ARCHITECTURES={sms}", - f"-DCMAKE_CUDA_COMPILER={env['CUDACXX']}", - ], + configure_command, sample_dir, - env, + cmake_env, timeout, ) if configure_result.returncode: @@ -393,6 +403,7 @@ def _compile_sample( str(jobs), ] search_root = cmake_build_dir + env = cmake_env else: self._run_build_command(["make", "clean"], sample_dir, env, timeout) command = [ diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index 1aadaf95..44164d1f 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -174,6 +174,44 @@ def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): assert 'PROPERTY CUDA_ARCHITECTURES "ivcore11"' in wrapper +def test_metax_cmake_wrapper_owns_toolchain_detection(tmp_path, monkeypatch): + sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + build_dir = tmp_path / "build" + binary = build_dir / "build" / "vectorAdd" + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs["env"])) + if "--build" in command: + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("binary", encoding="utf-8") + binary.chmod(0o755) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + cmake_command = "/opt/maca/tools/cu-bridge/tools/cmake_maca" + + result = CompatibilityAdapter()._compile_sample( + sample_dir, + build_dir, + { + "SMS": "70", + "CUDACXX": "/opt/maca/tools/cu-bridge/bin/cucc", + "CMAKE_COMMAND": cmake_command, + }, + 30, + 4, + "cmake", + ) + + configure_command, configure_env = calls[0] + assert result == binary + assert not any(arg.startswith("-DCMAKE_CUDA_") for arg in configure_command) + assert "CUDACXX" not in configure_env + assert configure_env["WCUDA_HOME"] == str(build_dir / "cmake-maca") + assert calls[1][1]["WCUDA_HOME"] == configure_env["WCUDA_HOME"] + + @pytest.mark.parametrize("platform", ["metax", "corex"]) def test_vendor_make_args_remove_nvidia_only_flags(platform): args = " ".join(CUDA_SAMPLE_CONFIGS[platform]["make_args"]) From d852fc7bf6d8fee418732671d844ad56f8ce33b1 Mon Sep 17 00:00:00 2001 From: Florent Li <1508269885@qq.com> Date: Mon, 10 Aug 2026 17:20:22 +0800 Subject: [PATCH 7/7] test: simplify compatibility adapter coverage --- tests/test_compatibility_adapter.py | 277 ++++++++-------------------- 1 file changed, 74 insertions(+), 203 deletions(-) diff --git a/tests/test_compatibility_adapter.py b/tests/test_compatibility_adapter.py index 44164d1f..17e7402c 100644 --- a/tests/test_compatibility_adapter.py +++ b/tests/test_compatibility_adapter.py @@ -12,7 +12,12 @@ from infinibench.dispatcher import Dispatcher -def _sample(root: Path, category: str, name: str, manifest: str) -> Path: +def _sample( + root: Path, + name: str, + manifest: str = "CMakeLists.txt", + category: str = "0_Introduction", +) -> Path: sample_dir = root / "Samples" / category / name sample_dir.mkdir(parents=True) content = f"project({name})\n" if manifest == "CMakeLists.txt" else "# test\n" @@ -28,62 +33,65 @@ def compiler_available(monkeypatch): ) +def _mock_successful_build(monkeypatch, binary): + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + if "--build" in command or (command[0] == "make" and "clean" not in command): + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("binary", encoding="utf-8") + binary.chmod(0o755) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + return calls + + def test_default_cuda_samples_dir_uses_repository_submodule(): repository_root = Path(__file__).resolve().parents[1] assert _CUDA_SAMPLES_DIR == repository_root / "submodules" / "cuda-samples" -def test_discovers_nested_cmake_and_make_samples(tmp_path): +def test_discovers_only_standalone_cmake_and_make_samples(tmp_path): (tmp_path / "Samples").mkdir() (tmp_path / "Samples" / "CMakeLists.txt").write_text("# aggregate\n") category = tmp_path / "Samples" / "0_Introduction" category.mkdir() (category / "CMakeLists.txt").write_text("# aggregate\n") - vector_add = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") - clock = _sample(tmp_path, "0_Introduction", "clock", "Makefile") + vector_add = _sample(tmp_path, "vectorAdd") + clock = _sample(tmp_path, "clock", "Makefile") - discovered = CompatibilityAdapter()._discover_sample_dirs( - tmp_path / "Samples", None - ) - - assert discovered == [clock, vector_add] - - -def test_sample_filter_rejects_unknown_names(tmp_path): - _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") - - with pytest.raises(ValueError, match="missingSample"): - CompatibilityAdapter()._discover_sample_dirs( - tmp_path / "Samples", ["vectorAdd", "missingSample"] - ) - - -def test_discovery_excludes_nested_cmake_group_manifests(tmp_path): group = tmp_path / "Samples" / "8_Platform_Specific" / "Tegra" group.mkdir(parents=True) (group / "CMakeLists.txt").write_text( "add_subdirectory(simpleGLES)\n", encoding="utf-8" ) - sample = _sample( + simple_gles = _sample( tmp_path, - "8_Platform_Specific/Tegra", "simpleGLES", - "CMakeLists.txt", + category="8_Platform_Specific/Tegra", ) discovered = CompatibilityAdapter()._discover_sample_dirs( tmp_path / "Samples", None ) - assert discovered == [sample] + assert discovered == [clock, vector_add, simple_gles] -def test_sample_filter_rejects_an_empty_list(tmp_path): - _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") +@pytest.mark.parametrize( + ("sample_filter", "error"), + [([], "non-empty"), (["vectorAdd", "missingSample"], "missingSample")], +) +def test_sample_filter_rejects_invalid_lists(tmp_path, sample_filter, error): + _sample(tmp_path, "vectorAdd") - with pytest.raises(ValueError, match="non-empty"): - CompatibilityAdapter()._discover_sample_dirs(tmp_path / "Samples", []) + with pytest.raises(ValueError, match=error): + CompatibilityAdapter()._discover_sample_dirs( + tmp_path / "Samples", sample_filter + ) @pytest.mark.parametrize("sms", ["", '80")\nmessage(FATAL_ERROR injected)']) @@ -92,49 +100,14 @@ def test_architecture_rejects_empty_or_cmake_control_characters(sms): CompatibilityAdapter._validate_architectures(sms) -@pytest.mark.parametrize( - ("build_system", "manifest", "sms", "compiler", "cmake_command", "jobs"), - [ - ( - "cmake", - "CMakeLists.txt", - "80", - "/usr/local/cuda/bin/nvcc", - "/vendor/bin/cmake_maca", - 7, - ), - ("make", "Makefile", "70", "/usr/local/musa/bin/mcc", "cmake", 3), - ], -) -def test_build_commands_include_compiler_arch_and_jobs( - tmp_path, - monkeypatch, - build_system, - manifest, - sms, - compiler, - cmake_command, - jobs, -): - sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", manifest) +def test_cmake_build_includes_compiler_arch_and_jobs(tmp_path, monkeypatch): + sample_dir = _sample(tmp_path, "vectorAdd") build_dir = tmp_path / "build" - expected_binary = ( - build_dir / "build" if build_system == "cmake" else sample_dir - ) / "vectorAdd" - calls = [] - - def fake_run(command, **kwargs): - calls.append(command) - is_build = (len(command) > 1 and command[1] == "--build") or ( - command[0] == "make" and "clean" not in command - ) - if is_build: - expected_binary.parent.mkdir(parents=True, exist_ok=True) - expected_binary.write_text("binary", encoding="utf-8") - expected_binary.chmod(0o755) - return subprocess.CompletedProcess(command, 0, "", "") - - monkeypatch.setattr(subprocess, "run", fake_run) + compiler = "/usr/local/cuda/bin/nvcc" + cmake_command = "/usr/bin/cmake" + sms, jobs = "80", 4 + expected_binary = build_dir / "build" / "vectorAdd" + calls = _mock_successful_build(monkeypatch, expected_binary) binary = CompatibilityAdapter()._compile_sample( sample_dir, build_dir, @@ -145,26 +118,17 @@ def fake_run(command, **kwargs): }, 30, jobs, - build_system, + "cmake", ) - expected = ( - [ - cmake_command, - "--build", - str(build_dir / "build"), - "--parallel", - str(jobs), - ] - if build_system == "cmake" - else ["make", f"-j{jobs}", f"SMS={sms}", f"NVCC={compiler}"] - ) assert binary == expected_binary - assert expected in calls + assert f"-DCMAKE_CUDA_ARCHITECTURES={sms}" in calls[0][0] + assert f"-DCMAKE_CUDA_COMPILER={compiler}" in calls[0][0] + assert calls[1][0][-2:] == ["--parallel", str(jobs)] def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): - sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + sample_dir = _sample(tmp_path, "vectorAdd") wrapper_dir = tmp_path / "wrapper" CompatibilityAdapter._write_cmake_wrapper(wrapper_dir, sample_dir, "ivcore11") @@ -175,20 +139,10 @@ def test_cmake_wrapper_overrides_sample_target_architectures(tmp_path): def test_metax_cmake_wrapper_owns_toolchain_detection(tmp_path, monkeypatch): - sample_dir = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") + sample_dir = _sample(tmp_path, "vectorAdd") build_dir = tmp_path / "build" binary = build_dir / "build" / "vectorAdd" - calls = [] - - def fake_run(command, **kwargs): - calls.append((command, kwargs["env"])) - if "--build" in command: - binary.parent.mkdir(parents=True, exist_ok=True) - binary.write_text("binary", encoding="utf-8") - binary.chmod(0o755) - return subprocess.CompletedProcess(command, 0, "", "") - - monkeypatch.setattr(subprocess, "run", fake_run) + calls = _mock_successful_build(monkeypatch, binary) cmake_command = "/opt/maca/tools/cu-bridge/tools/cmake_maca" result = CompatibilityAdapter()._compile_sample( @@ -204,26 +158,13 @@ def fake_run(command, **kwargs): "cmake", ) - configure_command, configure_env = calls[0] + configure_command, configure_kwargs = calls[0] + configure_env = configure_kwargs["env"] assert result == binary assert not any(arg.startswith("-DCMAKE_CUDA_") for arg in configure_command) assert "CUDACXX" not in configure_env assert configure_env["WCUDA_HOME"] == str(build_dir / "cmake-maca") - assert calls[1][1]["WCUDA_HOME"] == configure_env["WCUDA_HOME"] - - -@pytest.mark.parametrize("platform", ["metax", "corex"]) -def test_vendor_make_args_remove_nvidia_only_flags(platform): - args = " ".join(CUDA_SAMPLE_CONFIGS[platform]["make_args"]) - - assert all(flag not in args for flag in ("--threads", "-gencode", "-m64")) - - -def test_corex_make_args_keep_source_language_out_of_link_step(): - compile_args, link_args, _ = CUDA_SAMPLE_CONFIGS["corex"]["make_args"] - - assert "-x ivcore" in compile_args - assert "-x ivcore" not in link_args + assert calls[1][1]["env"]["WCUDA_HOME"] == configure_env["WCUDA_HOME"] def test_corex_defaults_target_bi_v150(): @@ -235,61 +176,21 @@ def test_corex_defaults_target_bi_v150(): assert "--cuda-gpu-arch=ivcore11" in link_args -def test_compile_env_matches_selected_vendor_toolkit(monkeypatch, compiler_available): - compiler = "/usr/local/corex/bin/clang++" - monkeypatch.setenv("CUDA_HOME", "/usr/local/cuda") - monkeypatch.setenv("CUDA_PATH", "/usr/local/cuda") - env = CompatibilityAdapter()._build_compile_env("corex", "ivcore11", compiler) - - expected_root = str(Path(compiler).resolve().parent.parent) - assert env["CUDACXX"] == compiler - assert env["CUDA_HOME"] == expected_root - assert env["CUDA_PATH"] == expected_root - - -@pytest.mark.parametrize( - ("compiler", "root_parent_index"), - [ - ("/opt/maca/tools/cu-bridge/bin/cucc", 3), - ("/opt/maca/mxgpu_llvm/bin/mxcc", 2), - ], -) -def test_metax_compile_env_infers_maca_path( - monkeypatch, compiler_available, compiler, root_parent_index -): - monkeypatch.delenv("MACA_PATH", raising=False) - - env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) - - assert env["MACA_PATH"] == str(Path(compiler).resolve().parents[root_parent_index]) - - -def test_metax_compile_env_preserves_explicit_maca_path( - monkeypatch, compiler_available -): - compiler = "/opt/maca/tools/cu-bridge/bin/cucc" - monkeypatch.setenv("MACA_PATH", "/custom/maca") - env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) - - assert env["MACA_PATH"] == "/custom/maca" - - -def test_metax_compile_env_selects_cu_bridge_cmake(monkeypatch, compiler_available): +def test_metax_compile_env_selects_wrapper(monkeypatch, compiler_available): compiler = "/opt/maca/tools/cu-bridge/bin/cucc" + monkeypatch.delenv("MACA_PATH", raising=False) env = CompatibilityAdapter()._build_compile_env("metax", "70", compiler) cmake_command = "/opt/maca/tools/cu-bridge/tools/cmake_maca" + assert env["MACA_PATH"] == str(Path(compiler).resolve().parents[3]) assert env["CMAKE_COMMAND"] == cmake_command assert env["CUCC_CMAKE_ENTRY"] == "2" assert env["PATH"].split(os.pathsep)[0] == str(Path(cmake_command).resolve().parent) def test_cuda_sample_metrics_keep_skips_separate(tmp_path, monkeypatch): - samples = [ - _sample(tmp_path, "0_Introduction", name, "CMakeLists.txt") - for name in ("passes", "fails", "waived") - ] + samples = [_sample(tmp_path, name) for name in ("passes", "fails", "waived")] binaries = {sample.name: tmp_path / f"{sample.name}.bin" for sample in samples} adapter = CompatibilityAdapter() @@ -325,9 +226,8 @@ def test_cuda_sample_metrics_keep_skips_separate(tmp_path, monkeypatch): assert values["compatibility.cuda_samples.run_pass_rate"] == 33.33 -@pytest.mark.parametrize("failure_stage", ["compile", "run"]) -def test_sample_failures_are_accounted_for(tmp_path, monkeypatch, failure_stage): - sample = _sample(tmp_path, "0_Introduction", "vectorAdd", "CMakeLists.txt") +def test_compile_failures_are_accounted_for(tmp_path, monkeypatch): + sample = _sample(tmp_path, "vectorAdd") adapter = CompatibilityAdapter() monkeypatch.setattr(adapter, "_discover_sample_dirs", lambda *_: [sample]) monkeypatch.setattr( @@ -337,15 +237,9 @@ def test_sample_failures_are_accounted_for(tmp_path, monkeypatch, failure_stage) ) def compile_sample(*_): - if failure_stage == "compile": - raise RuntimeError("compiler failed") - return tmp_path / "vectorAdd" - - def run_sample(*_): - raise RuntimeError("runtime failed") + raise RuntimeError("compiler failed") monkeypatch.setattr(adapter, "_compile_sample", compile_sample) - monkeypatch.setattr(adapter, "_run_sample", run_sample) metrics = adapter._run_cuda_samples_test( {"platform": "nvidia", "cuda_samples_dir": str(tmp_path)} @@ -353,31 +247,25 @@ def run_sample(*_): values = {metric["name"]: metric["value"] for metric in metrics} details = values["compatibility.cuda_samples.details"] - compile_passed = int(failure_stage == "run") - assert values["compatibility.cuda_samples.compile_passed"] == compile_passed - assert values["compatibility.cuda_samples.run_failed"] == compile_passed + assert values["compatibility.cuda_samples.compile_passed"] == 0 + assert values["compatibility.cuda_samples.run_failed"] == 0 assert values["compatibility.cuda_samples.run_skipped"] == 0 - assert details[0]["compile_result"] == ("pass" if compile_passed else "fail") - assert details[0]["run_result"] == ("fail" if compile_passed else "not_run") + assert details[0]["compile_result"] == "fail" + assert details[0]["run_result"] == "not_run" -def test_run_sample_classifies_cuda_waiver(tmp_path): - binary = tmp_path / "sample" - binary.write_text("binary", encoding="utf-8") - binary.chmod(0o755) - adapter = CompatibilityAdapter() - - result, error = adapter._classify_run_result( - subprocess.CompletedProcess([str(binary)], 2, "Sample waived", "") +def test_run_sample_classifies_cuda_waiver(): + result, error = CompatibilityAdapter._classify_run_result( + subprocess.CompletedProcess(["sample"], 2, "Sample waived", "") ) assert result == "skip" assert "waived" in error.lower() -def test_run_sample_uses_the_binary_directory(tmp_path, monkeypatch): - binary = tmp_path / "build" / "sample" - binary.parent.mkdir() +def test_run_sample_resolves_path_and_uses_binary_directory(tmp_path, monkeypatch): + binary = Path("build") / "sample" + monkeypatch.chdir(tmp_path) calls = [] def fake_run(command, **kwargs): @@ -388,12 +276,13 @@ def fake_run(command, **kwargs): result, _ = CompatibilityAdapter()._run_sample(binary, 30) + resolved_binary = binary.resolve() assert result == "pass" assert calls == [ ( - [str(binary)], + [str(resolved_binary)], { - "cwd": str(binary.parent), + "cwd": str(resolved_binary.parent), "capture_output": True, "text": True, "errors": "replace", @@ -404,24 +293,6 @@ def fake_run(command, **kwargs): ] -def test_run_sample_resolves_a_relative_binary_path(tmp_path, monkeypatch): - binary = Path("build") / "sample" - monkeypatch.chdir(tmp_path) - calls = [] - - def fake_run(command, **kwargs): - calls.append((command, kwargs["cwd"])) - return subprocess.CompletedProcess(command, 0, "Result = PASS", "") - - monkeypatch.setattr(subprocess, "run", fake_run) - - result, _ = CompatibilityAdapter()._run_sample(binary, 30) - - resolved_binary = binary.resolve() - assert result == "pass" - assert calls == [([str(resolved_binary)], str(resolved_binary.parent))] - - def test_metrics_reject_an_empty_sample_set(): with pytest.raises(ValueError, match="No CUDA samples selected"): CompatibilityAdapter._build_metrics([])