From 5ab81e0bb081a2f7ba502f5e07cadc0cdacfcc31 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:45:00 +0000 Subject: [PATCH 1/4] Refuse an operand on another device than the operator's data Applying a CartesianSense built on host tensors to a CUDA tensor, or solving with it, crashed the process. Operator application and the linear solve now raise a ValueError naming both devices. The MoDL lesson keeps the operator, its data and the network on one device, the card when there is one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QLvu9frE7HdzPd4yCU --- .../examples/06-learning/02-modl-with-admm.py | 19 ++++++--- src/bartorch/_operator.py | 41 +++++++++++++++++++ src/bartorch/optim/_linear.py | 5 ++- tests/test_cuda.py | 41 +++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/docs/examples/06-learning/02-modl-with-admm.py b/docs/examples/06-learning/02-modl-with-admm.py index c85acec7f..1e7d5d5b9 100644 --- a/docs/examples/06-learning/02-modl-with-admm.py +++ b/docs/examples/06-learning/02-modl-with-admm.py @@ -103,7 +103,7 @@ def outline(axis, crop): def scaled(estimate, reference): """``estimate`` scaled to ``reference`` in the least-squares sense.""" - a, b = estimate.abs().double(), reference.abs().double() + a, b = estimate.detach().abs().double().cpu(), reference.detach().abs().double().cpu() return (float((a * b).sum() / (a * a).sum()) * a).float() @@ -202,6 +202,11 @@ def _slice_image(index): _ = torch.manual_seed(0) +# The operator, its data and the network are on one device: the card when +# there is one. +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"device: {DEVICE}") + train_images = images[:24] valid_images = images[24:] print(f"{len(train_images)} slices to train, {len(valid_images)} to validate") @@ -229,7 +234,8 @@ def _slice_image(index): lines = torch.rand(SIZE, generator=torch.Generator().manual_seed(1)) < density lines[SIZE // 2 - CENTRE // 2 : SIZE // 2 + CENTRE // 2] = True # sphinx_gallery_end_ignore -pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous() +pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous().to(DEVICE) +sensitivities = sensitivities.to(DEVICE) A = linop.CartesianSense(sensitivities, (SIZE, SIZE), pattern=pattern) print(f"{int(lines.sum())} of {SIZE} phase encodes") @@ -246,9 +252,9 @@ def _slice_image(index): def measure(images, generator=None): """Images, their noisy k-space and the adjoint reconstructions.""" - x = torch.stack(list(images)) + x = torch.stack(list(images)).to(DEVICE) y = torch.stack([A(item) for item in x]) - noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator) + noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator).to(DEVICE) y = y + NOISE * noise return x, y, torch.stack([A.H(item) for item in y]) @@ -406,7 +412,7 @@ def configure_optimizers(self): trainer = lightning.Trainer( max_epochs=EPOCHS, - accelerator="cpu", + accelerator="gpu" if DEVICE.type == "cuda" else "cpu", logger=False, enable_checkpointing=False, enable_model_summary=False, @@ -414,6 +420,7 @@ def configure_optimizers(self): ) trainer.fit(Reconstruction(model), train_loader, valid_loader) +model.to(DEVICE) print(f"rho ended at {float(block.rho.detach()):.3f}") # %% @@ -454,7 +461,7 @@ def quality(estimate): print(f"{name:>16} PSNR {made[0]:5.2f} dB SSIM {made[1]:.3f}") # sphinx_gallery_start_ignore -reference = truth[0] +reference = truth[0].cpu() top = float(reference.abs().max()) learned_name = f"MoDL, K={ITERATIONS}" zoom = (slice(70, 118), slice(40, 88)) diff --git a/src/bartorch/_operator.py b/src/bartorch/_operator.py index 056f9444d..c5b6fcaa1 100644 --- a/src/bartorch/_operator.py +++ b/src/bartorch/_operator.py @@ -106,6 +106,46 @@ def axes_flags(axes, ndim: int) -> int: return _axes_to_flags(axes, ndim) +def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None: + """Raise if ``x`` is not on ``held``, the device an operator's own data is on. + + BART dereferences an operand and the data an operator holds as if both + were where the operator was built, so a host operator handed a device + tensor reads device memory from the host. + + Parameters + ---------- + held : torch.device or None + Where the operator's data is; ``None`` for an operator that holds none. + x : torch.Tensor + The operand. + what : str + The operand's name in the message. + + Raises + ------ + ValueError + When ``x`` is on another device. + """ + if held is None or same_device(held, x.device): + return + raise ValueError( + f"{what} is on {x.device} but the operator was built on {held}; " + f"move one of them with .to({str(x.device)!r}) or .to({str(held)!r})" + ) + + +def same_device(a: torch.device, b: torch.device) -> bool: + """Whether ``a`` and ``b`` name one device, with a bare ``cuda`` as the current card.""" + a, b = torch.device(a), torch.device(b) + if a.type != b.type: + return False + if a.type != "cuda": + return True + current = torch.cuda.current_device() if a.index is None or b.index is None else None + return (current if a.index is None else a.index) == (current if b.index is None else b.index) + + def as_operand(x: Any, shape: Shape, what: str) -> torch.Tensor: """A contiguous complex64 tensor of ``shape``, on a device BART can reach.""" if not isinstance(x, torch.Tensor): @@ -288,6 +328,7 @@ def _apply( f"{type(self).__name__} defines neither _create nor this method" ) x = as_operand(x, ishape, "input") + check_device(self.device, x, "input") if out is None: y = torch.empty(oshape, dtype=torch.complex64, device=x.device) else: diff --git a/src/bartorch/optim/_linear.py b/src/bartorch/optim/_linear.py index ce388c960..30f9a5a00 100644 --- a/src/bartorch/optim/_linear.py +++ b/src/bartorch/optim/_linear.py @@ -17,7 +17,7 @@ from bartorch import _marshal from bartorch._dispatch import BartError, _ensure_ready, _lock, _on_device from bartorch._lib import library -from bartorch._operator import as_operand +from bartorch._operator import as_operand, check_device, same_device from bartorch.optim._blocks import ( ADMMBlock, FISTABlock, @@ -335,10 +335,13 @@ def _solve( appended to it.""" op = A._bart() y = as_operand(y, op.oshape, "y") + check_device(op.device, y, "y") if x0 is None: x = torch.zeros(op.ishape, dtype=torch.complex64, device=y.device) else: x = as_operand(x0, op.ishape, "x0").clone() + if not same_device(x.device, y.device): + raise ValueError(f"x0 is on {x.device} but y is on {y.device}") _ensure_ready() lib = library() diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 69dd9c3f5..0296061ba 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -169,3 +169,44 @@ def test_a_tool_that_is_not_kept_on_the_card_still_answers_on_it(): assert out.device.type == "cuda" expected = bartorch.fftmod(img.cpu(), (-2, -1)) torch.testing.assert_close(out.cpu(), expected, rtol=1e-4, atol=1e-5) + + +def _sense_on(device: str, n: int = 16, coils: int = 2) -> linop.CartesianSense: + sensitivities = torch.ones(coils, n, n, dtype=torch.complex64, device=device) + pattern = torch.ones(n, n, dtype=torch.complex64, device=device) + return linop.CartesianSense(sensitivities, (n, n), pattern=pattern) + + +def test_an_operand_on_another_device_than_the_operator_is_refused_before_bart_sees_it(): + A = _sense_on("cpu") + A._h # noqa: B018 -- builds the operator, which records where its data is + A.device = torch.device("cuda", 0) + x = torch.randn(A.ishape, dtype=torch.complex64) + y = torch.randn(A.oshape, dtype=torch.complex64) + with pytest.raises(ValueError, match="built on cuda:0"): + A(x) + with pytest.raises(ValueError, match="built on cuda:0"): + A.H(y) + with pytest.raises(ValueError, match="built on cuda:0"): + bartorch.optim.cg(y, A, maxiter=2) + + +@requires_cuda +@pytest.mark.parametrize(("built", "operand"), [("cpu", "cuda"), ("cuda", "cpu")]) +def test_mixing_host_and_device_tensors_raises_instead_of_crashing(built, operand): + A = _sense_on(built) + x = torch.randn(A.ishape, dtype=torch.complex64, device=operand) + y = torch.randn(A.oshape, dtype=torch.complex64, device=operand) + with pytest.raises(ValueError, match="move one of them"): + A(x) + with pytest.raises(ValueError, match="move one of them"): + A.H(y) + with pytest.raises(ValueError, match="move one of them"): + bartorch.optim.cg(y, A, maxiter=2) + + +@requires_cuda +def test_a_bare_cuda_device_is_the_current_card(): + A = _sense_on("cuda") + x = torch.randn(A.ishape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}") + assert A(x).device == x.device From 223672f96c5d4c7c5b15a5e5182600ae225071cd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:54:02 +0000 Subject: [PATCH 2/4] Device-mismatch test on the host covers operator application only Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QLvu9frE7HdzPd4yCU --- tests/test_cuda.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 0296061ba..de542cdf3 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -182,13 +182,8 @@ def test_an_operand_on_another_device_than_the_operator_is_refused_before_bart_s A._h # noqa: B018 -- builds the operator, which records where its data is A.device = torch.device("cuda", 0) x = torch.randn(A.ishape, dtype=torch.complex64) - y = torch.randn(A.oshape, dtype=torch.complex64) with pytest.raises(ValueError, match="built on cuda:0"): A(x) - with pytest.raises(ValueError, match="built on cuda:0"): - A.H(y) - with pytest.raises(ValueError, match="built on cuda:0"): - bartorch.optim.cg(y, A, maxiter=2) @requires_cuda From 31b0474c86900a652a11bf18809b6e51abcd73b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 17:10:50 +0000 Subject: [PATCH 3/4] An operator built on a card takes host operands; only a host operator refuses a device operand Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QLvu9frE7HdzPd4yCU --- src/bartorch/_operator.py | 16 +++++++------- tests/test_cuda.py | 46 ++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/bartorch/_operator.py b/src/bartorch/_operator.py index c5b6fcaa1..cac109e2e 100644 --- a/src/bartorch/_operator.py +++ b/src/bartorch/_operator.py @@ -107,11 +107,11 @@ def axes_flags(axes, ndim: int) -> int: def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None: - """Raise if ``x`` is not on ``held``, the device an operator's own data is on. + """Raise if ``x`` is on a card and the operator's data is on the host. - BART dereferences an operand and the data an operator holds as if both - were where the operator was built, so a host operator handed a device - tensor reads device memory from the host. + BART dereferences the data an operator holds where its operand is, so a + host operator handed a device tensor reads host memory from the card. An + operator built on a card takes host operands: it moves them itself. Parameters ---------- @@ -125,13 +125,13 @@ def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None: Raises ------ ValueError - When ``x`` is on another device. + When ``x`` is on a card and ``held`` is the host. """ - if held is None or same_device(held, x.device): + if held is None or torch.device(held).type != "cpu" or x.device.type == "cpu": return raise ValueError( - f"{what} is on {x.device} but the operator was built on {held}; " - f"move one of them with .to({str(x.device)!r}) or .to({str(held)!r})" + f"{what} is on {x.device} but the operator was built on the host; " + f"build it on {x.device} or move {what} to the host with .cpu()" ) diff --git a/tests/test_cuda.py b/tests/test_cuda.py index de542cdf3..4f7be08f4 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -12,6 +12,7 @@ import bartorch._reference as ref import bartorch.tools as bt from bartorch import linop +from bartorch._operator import check_device requires_cuda = pytest.mark.skipif( not bartorch._cuda.available(), reason="no CUDA device, or the library was built without CUDA" @@ -177,31 +178,40 @@ def _sense_on(device: str, n: int = 16, coils: int = 2) -> linop.CartesianSense: return linop.CartesianSense(sensitivities, (n, n), pattern=pattern) -def test_an_operand_on_another_device_than_the_operator_is_refused_before_bart_sees_it(): - A = _sense_on("cpu") - A._h # noqa: B018 -- builds the operator, which records where its data is - A.device = torch.device("cuda", 0) - x = torch.randn(A.ishape, dtype=torch.complex64) - with pytest.raises(ValueError, match="built on cuda:0"): - A(x) +def test_a_device_operand_for_a_host_operator_is_refused_before_bart_sees_it(): + x = torch.empty(4, dtype=torch.complex64, device="meta") + with pytest.raises(ValueError, match="built on the host"): + check_device(torch.device("cpu"), x, "input") + + +def test_an_operator_holding_no_data_takes_an_operand_anywhere(): + check_device(None, torch.empty(4, dtype=torch.complex64, device="meta"), "input") @requires_cuda -@pytest.mark.parametrize(("built", "operand"), [("cpu", "cuda"), ("cuda", "cpu")]) -def test_mixing_host_and_device_tensors_raises_instead_of_crashing(built, operand): - A = _sense_on(built) - x = torch.randn(A.ishape, dtype=torch.complex64, device=operand) - y = torch.randn(A.oshape, dtype=torch.complex64, device=operand) - with pytest.raises(ValueError, match="move one of them"): +def test_a_device_operand_for_a_host_operator_raises_instead_of_crashing(): + A = _sense_on("cpu") + x = torch.randn(A.ishape, dtype=torch.complex64, device="cuda") + y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda") + with pytest.raises(ValueError, match="built on the host"): A(x) - with pytest.raises(ValueError, match="move one of them"): + with pytest.raises(ValueError, match="built on the host"): A.H(y) - with pytest.raises(ValueError, match="move one of them"): + with pytest.raises(ValueError, match="built on the host"): bartorch.optim.cg(y, A, maxiter=2) @requires_cuda -def test_a_bare_cuda_device_is_the_current_card(): +def test_a_starting_image_on_the_bare_cuda_device_is_on_the_current_card(): + A = _sense_on("cuda") + y = torch.randn(A.oshape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}") + x0 = torch.zeros(A.ishape, dtype=torch.complex64, device="cuda") + assert bartorch.optim.cg(y, A, x0=x0, maxiter=2).device == y.device + + +@requires_cuda +def test_a_starting_image_on_another_device_than_the_data_is_refused(): A = _sense_on("cuda") - x = torch.randn(A.ishape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}") - assert A(x).device == x.device + y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda") + with pytest.raises(ValueError, match="x0 is on cpu"): + bartorch.optim.cg(y, A, x0=torch.zeros(A.ishape, dtype=torch.complex64), maxiter=2) From e9e420485d7a4c69c6104cb66e08de883f71c003 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 17:40:48 +0000 Subject: [PATCH 4/4] Compare the starting image's device with the data's directly Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QLvu9frE7HdzPd4yCU --- src/bartorch/_operator.py | 11 ----------- src/bartorch/optim/_linear.py | 4 ++-- tests/test_cuda.py | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/bartorch/_operator.py b/src/bartorch/_operator.py index cac109e2e..d2ff78597 100644 --- a/src/bartorch/_operator.py +++ b/src/bartorch/_operator.py @@ -135,17 +135,6 @@ def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None: ) -def same_device(a: torch.device, b: torch.device) -> bool: - """Whether ``a`` and ``b`` name one device, with a bare ``cuda`` as the current card.""" - a, b = torch.device(a), torch.device(b) - if a.type != b.type: - return False - if a.type != "cuda": - return True - current = torch.cuda.current_device() if a.index is None or b.index is None else None - return (current if a.index is None else a.index) == (current if b.index is None else b.index) - - def as_operand(x: Any, shape: Shape, what: str) -> torch.Tensor: """A contiguous complex64 tensor of ``shape``, on a device BART can reach.""" if not isinstance(x, torch.Tensor): diff --git a/src/bartorch/optim/_linear.py b/src/bartorch/optim/_linear.py index 30f9a5a00..3f5cdc82c 100644 --- a/src/bartorch/optim/_linear.py +++ b/src/bartorch/optim/_linear.py @@ -17,7 +17,7 @@ from bartorch import _marshal from bartorch._dispatch import BartError, _ensure_ready, _lock, _on_device from bartorch._lib import library -from bartorch._operator import as_operand, check_device, same_device +from bartorch._operator import as_operand, check_device from bartorch.optim._blocks import ( ADMMBlock, FISTABlock, @@ -340,7 +340,7 @@ def _solve( x = torch.zeros(op.ishape, dtype=torch.complex64, device=y.device) else: x = as_operand(x0, op.ishape, "x0").clone() - if not same_device(x.device, y.device): + if x.device != y.device: raise ValueError(f"x0 is on {x.device} but y is on {y.device}") _ensure_ready() diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 4f7be08f4..06d754c73 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -202,7 +202,7 @@ def test_a_device_operand_for_a_host_operator_raises_instead_of_crashing(): @requires_cuda -def test_a_starting_image_on_the_bare_cuda_device_is_on_the_current_card(): +def test_a_starting_image_made_on_the_bare_cuda_device_is_on_the_current_card(): A = _sense_on("cuda") y = torch.randn(A.oshape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}") x0 = torch.zeros(A.ishape, dtype=torch.complex64, device="cuda")