Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 6 additions & 44 deletions cuda_bindings/tests/nvml/conftest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from collections import namedtuple

import pytest
from cuda_python_test_helpers.arch_check import unsupported_before # noqa: F401
Expand All @@ -26,56 +25,19 @@ def nvml_init():
yield


@pytest.fixture(scope="session", autouse=True)
def device_info():
dev_count = None
bus_id_to_board_details = {}

with NVMLInitializer():
dev_count = nvml.device_get_count_v2()

# Store some details for each device now when we know NVML is in known state
for i in range(dev_count):
try:
dev = nvml.device_get_handle_by_index_v2(i)
except nvml.NoPermissionError:
continue
pci_info = nvml.device_get_pci_info_v3(dev)

name = nvml.device_get_name(dev)
# Get architecture name ex: Ampere, Kepler
arch_id = nvml.device_get_architecture(dev)

BoardCfg = namedtuple("BoardCfg", "name, ids_arr")
board = BoardCfg(name, ids_arr=[(pci_info.pci_device_id, pci_info.pci_sub_system_id)])

try:
serial = nvml.device_get_serial(dev)
except nvml.NvmlError:
serial = None

bus_id = pci_info.bus_id
device_id = pci_info.device_
uuid = nvml.device_get_uuid(dev)

BoardDetails = namedtuple("BoardDetails", "name, board, arch_id, bus_id, device_id, serial")
bus_id_to_board_details[uuid] = BoardDetails(name, board, arch_id, bus_id, device_id, serial)

return bus_id_to_board_details


def get_devices(device_info):
for uuid in list(device_info.keys()):
def get_devices():
dev_count = nvml.device_get_count_v2()
for i in range(dev_count):
try:
yield nvml.device_get_handle_by_uuid(uuid)
yield nvml.device_get_handle_by_index_v2(i)
except nvml.NoPermissionError:
continue # ignore devices that can't be accessed


@pytest.fixture
def all_devices(device_info):
def all_devices():
with NVMLInitializer():
yield sorted(set(get_devices(device_info)))
yield sorted(set(get_devices()))


@pytest.fixture
Expand Down
5 changes: 3 additions & 2 deletions cuda_bindings/tests/nvml/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ def test_cuda_device_order():
cuda_devices = get_cuda_device_names()
nvml_devices = get_nvml_device_names()

if any("Thor" in device["name"] for device in nvml_devices):
pytest.skip("Skipping test on Thor, which has non-standard device naming")
for kind in ("Orin", "Thor"):
if any(kind in device["name"] for device in nvml_devices):
pytest.skip(f"Skipping test on {kind}, which has non-standard device naming")
return

if "CUDA_VISIBLE_DEVICES" not in os.environ:
Expand Down
12 changes: 8 additions & 4 deletions cuda_bindings/tests/nvml/test_device.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,15 @@ def test_grid_licensable_features(all_devices):
nvml.GridLicenseExpiry(feature.license_expiry)


def test_get_handle_by_uuidv(all_devices):
def test_get_handle_by_uuidv(all_devices, subtests):
for device in all_devices:
uuid = nvml.device_get_uuid(device)
new_handle = nvml.device_get_handle_by_uuidv(nvml.UUIDType.ASCII, uuid.encode("ascii"))
assert new_handle == device
with subtests.test(device_index=nvml.device_get_index(device)):
uuid = nvml.device_get_uuid(device)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I asked codex to generate a fix for the one finding it had. The fix is here:

rwgk@c4519f2

The finding was:

  • Blocking: PR 2742 still leaves two Orin UUID failures unresolved. cuda_bindings/tests/nvml/test_device.py:69 passes a bare 36-character UUID to UUIDV, which requires 40 bytes and raises ValueError; unsupported_before catches neither that nor the observed NotFoundError. The legacy lookup in cuda_bindings/tests/nvml/test_pynvml.py:55 is unchanged and also returns NotFoundError on the target Orin.

if "Orin" in nvml.device_get_name(device) and len(uuid) == 36:
pytest.skip("UUID lookup is unsupported on Orin, which reports a UUID without a GPU- prefix")
with unsupported_before(device, None):
new_handle = nvml.device_get_handle_by_uuidv(nvml.UUIDType.ASCII, uuid.encode("ascii"))
assert new_handle == device


def test_get_nv_link_supported_bw_modes(all_devices, subtests):
Expand Down
14 changes: 9 additions & 5 deletions cuda_bindings/tests/nvml/test_pynvml.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,13 @@ def test_device_get_attributes(mig_handles):
pytest.skip("No MIG devices found")


def test_device_get_handle_by_uuid(ngpus, uuids):
handles = [nvml.device_get_handle_by_uuid(uuids[i]) for i in range(ngpus)]
assert len(handles) == ngpus
def test_device_get_handle_by_uuid(ngpus, handles, uuids, subtests):
for i in range(ngpus):
with subtests.test(device_index=i):
uuid = uuids[i]
if "Orin" in nvml.device_get_name(handles[i]) and len(uuid) == 36:
pytest.skip("UUID lookup is unsupported on Orin, which reports a UUID without a GPU- prefix")
assert nvml.device_get_handle_by_uuid(uuid) == handles[i]


def test_device_get_handle_by_pci_bus_id(ngpus, pci_info):
Expand All @@ -68,7 +72,7 @@ def test_device_get_memory_affinity(handles, scope, subtests):
size = 1024
for device_index, handle in enumerate(handles):
with subtests.test(device_index=device_index):
with unsupported_before(handle, nvml.DeviceArch.KEPLER):
with unsupported_before(handle, None):
node_set = nvml.device_get_memory_affinity(handle, size, scope)
assert node_set is not None
assert len(node_set) == size
Expand All @@ -80,7 +84,7 @@ def test_device_get_cpu_affinity_within_scope(handles, scope, subtests):
size = 1024
for device_index, handle in enumerate(handles):
with subtests.test(device_index=device_index):
with unsupported_before(handle, nvml.DeviceArch.KEPLER):
with unsupported_before(handle, None):
cpu_set = nvml.device_get_cpu_affinity_within_scope(handle, size, scope)
assert cpu_set is not None
assert len(cpu_set) == size
Expand Down
Loading