diff --git a/docs/examples/06-learning/02-modl-with-admm.py b/docs/examples/06-learning/02-modl-with-admm.py index c85acec7f..1e7d5d5b9 100644 --- a/docs/examples/06-learning/02-modl-with-admm.py +++ b/docs/examples/06-learning/02-modl-with-admm.py @@ -103,7 +103,7 @@ def outline(axis, crop): def scaled(estimate, reference): """``estimate`` scaled to ``reference`` in the least-squares sense.""" - a, b = estimate.abs().double(), reference.abs().double() + a, b = estimate.detach().abs().double().cpu(), reference.detach().abs().double().cpu() return (float((a * b).sum() / (a * a).sum()) * a).float() @@ -202,6 +202,11 @@ def _slice_image(index): _ = torch.manual_seed(0) +# The operator, its data and the network are on one device: the card when +# there is one. +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"device: {DEVICE}") + train_images = images[:24] valid_images = images[24:] print(f"{len(train_images)} slices to train, {len(valid_images)} to validate") @@ -229,7 +234,8 @@ def _slice_image(index): lines = torch.rand(SIZE, generator=torch.Generator().manual_seed(1)) < density lines[SIZE // 2 - CENTRE // 2 : SIZE // 2 + CENTRE // 2] = True # sphinx_gallery_end_ignore -pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous() +pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous().to(DEVICE) +sensitivities = sensitivities.to(DEVICE) A = linop.CartesianSense(sensitivities, (SIZE, SIZE), pattern=pattern) print(f"{int(lines.sum())} of {SIZE} phase encodes") @@ -246,9 +252,9 @@ def _slice_image(index): def measure(images, generator=None): """Images, their noisy k-space and the adjoint reconstructions.""" - x = torch.stack(list(images)) + x = torch.stack(list(images)).to(DEVICE) y = torch.stack([A(item) for item in x]) - noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator) + noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator).to(DEVICE) y = y + NOISE * noise return x, y, torch.stack([A.H(item) for item in y]) @@ -406,7 +412,7 @@ def configure_optimizers(self): trainer = lightning.Trainer( max_epochs=EPOCHS, - accelerator="cpu", + accelerator="gpu" if DEVICE.type == "cuda" else "cpu", logger=False, enable_checkpointing=False, enable_model_summary=False, @@ -414,6 +420,7 @@ def configure_optimizers(self): ) trainer.fit(Reconstruction(model), train_loader, valid_loader) +model.to(DEVICE) print(f"rho ended at {float(block.rho.detach()):.3f}") # %% @@ -454,7 +461,7 @@ def quality(estimate): print(f"{name:>16} PSNR {made[0]:5.2f} dB SSIM {made[1]:.3f}") # sphinx_gallery_start_ignore -reference = truth[0] +reference = truth[0].cpu() top = float(reference.abs().max()) learned_name = f"MoDL, K={ITERATIONS}" zoom = (slice(70, 118), slice(40, 88)) diff --git a/src/bartorch/_operator.py b/src/bartorch/_operator.py index 056f9444d..d2ff78597 100644 --- a/src/bartorch/_operator.py +++ b/src/bartorch/_operator.py @@ -106,6 +106,35 @@ def axes_flags(axes, ndim: int) -> int: return _axes_to_flags(axes, ndim) +def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None: + """Raise if ``x`` is on a card and the operator's data is on the host. + + BART dereferences the data an operator holds where its operand is, so a + host operator handed a device tensor reads host memory from the card. An + operator built on a card takes host operands: it moves them itself. + + Parameters + ---------- + held : torch.device or None + Where the operator's data is; ``None`` for an operator that holds none. + x : torch.Tensor + The operand. + what : str + The operand's name in the message. + + Raises + ------ + ValueError + When ``x`` is on a card and ``held`` is the host. + """ + if held is None or torch.device(held).type != "cpu" or x.device.type == "cpu": + return + raise ValueError( + f"{what} is on {x.device} but the operator was built on the host; " + f"build it on {x.device} or move {what} to the host with .cpu()" + ) + + def as_operand(x: Any, shape: Shape, what: str) -> torch.Tensor: """A contiguous complex64 tensor of ``shape``, on a device BART can reach.""" if not isinstance(x, torch.Tensor): @@ -288,6 +317,7 @@ def _apply( f"{type(self).__name__} defines neither _create nor this method" ) x = as_operand(x, ishape, "input") + check_device(self.device, x, "input") if out is None: y = torch.empty(oshape, dtype=torch.complex64, device=x.device) else: diff --git a/src/bartorch/optim/_linear.py b/src/bartorch/optim/_linear.py index ce388c960..3f5cdc82c 100644 --- a/src/bartorch/optim/_linear.py +++ b/src/bartorch/optim/_linear.py @@ -17,7 +17,7 @@ from bartorch import _marshal from bartorch._dispatch import BartError, _ensure_ready, _lock, _on_device from bartorch._lib import library -from bartorch._operator import as_operand +from bartorch._operator import as_operand, check_device from bartorch.optim._blocks import ( ADMMBlock, FISTABlock, @@ -335,10 +335,13 @@ def _solve( appended to it.""" op = A._bart() y = as_operand(y, op.oshape, "y") + check_device(op.device, y, "y") if x0 is None: x = torch.zeros(op.ishape, dtype=torch.complex64, device=y.device) else: x = as_operand(x0, op.ishape, "x0").clone() + if x.device != y.device: + raise ValueError(f"x0 is on {x.device} but y is on {y.device}") _ensure_ready() lib = library() diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 69dd9c3f5..06d754c73 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -12,6 +12,7 @@ import bartorch._reference as ref import bartorch.tools as bt from bartorch import linop +from bartorch._operator import check_device requires_cuda = pytest.mark.skipif( not bartorch._cuda.available(), reason="no CUDA device, or the library was built without CUDA" @@ -169,3 +170,48 @@ def test_a_tool_that_is_not_kept_on_the_card_still_answers_on_it(): assert out.device.type == "cuda" expected = bartorch.fftmod(img.cpu(), (-2, -1)) torch.testing.assert_close(out.cpu(), expected, rtol=1e-4, atol=1e-5) + + +def _sense_on(device: str, n: int = 16, coils: int = 2) -> linop.CartesianSense: + sensitivities = torch.ones(coils, n, n, dtype=torch.complex64, device=device) + pattern = torch.ones(n, n, dtype=torch.complex64, device=device) + return linop.CartesianSense(sensitivities, (n, n), pattern=pattern) + + +def test_a_device_operand_for_a_host_operator_is_refused_before_bart_sees_it(): + x = torch.empty(4, dtype=torch.complex64, device="meta") + with pytest.raises(ValueError, match="built on the host"): + check_device(torch.device("cpu"), x, "input") + + +def test_an_operator_holding_no_data_takes_an_operand_anywhere(): + check_device(None, torch.empty(4, dtype=torch.complex64, device="meta"), "input") + + +@requires_cuda +def test_a_device_operand_for_a_host_operator_raises_instead_of_crashing(): + A = _sense_on("cpu") + x = torch.randn(A.ishape, dtype=torch.complex64, device="cuda") + y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda") + with pytest.raises(ValueError, match="built on the host"): + A(x) + with pytest.raises(ValueError, match="built on the host"): + A.H(y) + with pytest.raises(ValueError, match="built on the host"): + bartorch.optim.cg(y, A, maxiter=2) + + +@requires_cuda +def test_a_starting_image_made_on_the_bare_cuda_device_is_on_the_current_card(): + A = _sense_on("cuda") + y = torch.randn(A.oshape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}") + x0 = torch.zeros(A.ishape, dtype=torch.complex64, device="cuda") + assert bartorch.optim.cg(y, A, x0=x0, maxiter=2).device == y.device + + +@requires_cuda +def test_a_starting_image_on_another_device_than_the_data_is_refused(): + A = _sense_on("cuda") + y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda") + with pytest.raises(ValueError, match="x0 is on cpu"): + bartorch.optim.cg(y, A, x0=torch.zeros(A.ishape, dtype=torch.complex64), maxiter=2)