Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions docs/examples/06-learning/02-modl-with-admm.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def outline(axis, crop):

def scaled(estimate, reference):
"""``estimate`` scaled to ``reference`` in the least-squares sense."""
a, b = estimate.abs().double(), reference.abs().double()
a, b = estimate.detach().abs().double().cpu(), reference.detach().abs().double().cpu()
return (float((a * b).sum() / (a * a).sum()) * a).float()


Expand Down Expand Up @@ -202,6 +202,11 @@ def _slice_image(index):

_ = torch.manual_seed(0)

# The operator, its data and the network are on one device: the card when
# there is one.
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"device: {DEVICE}")

train_images = images[:24]
valid_images = images[24:]
print(f"{len(train_images)} slices to train, {len(valid_images)} to validate")
Expand Down Expand Up @@ -229,7 +234,8 @@ def _slice_image(index):
lines = torch.rand(SIZE, generator=torch.Generator().manual_seed(1)) < density
lines[SIZE // 2 - CENTRE // 2 : SIZE // 2 + CENTRE // 2] = True
# sphinx_gallery_end_ignore
pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous()
pattern = lines.to(torch.complex64)[:, None].expand(SIZE, SIZE).contiguous().to(DEVICE)
sensitivities = sensitivities.to(DEVICE)

A = linop.CartesianSense(sensitivities, (SIZE, SIZE), pattern=pattern)
print(f"{int(lines.sum())} of {SIZE} phase encodes")
Expand All @@ -246,9 +252,9 @@ def _slice_image(index):

def measure(images, generator=None):
"""Images, their noisy k-space and the adjoint reconstructions."""
x = torch.stack(list(images))
x = torch.stack(list(images)).to(DEVICE)
y = torch.stack([A(item) for item in x])
noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator)
noise = torch.randn(y.shape, dtype=torch.complex64, generator=generator).to(DEVICE)
y = y + NOISE * noise
return x, y, torch.stack([A.H(item) for item in y])

Expand Down Expand Up @@ -406,14 +412,15 @@ def configure_optimizers(self):

trainer = lightning.Trainer(
max_epochs=EPOCHS,
accelerator="cpu",
accelerator="gpu" if DEVICE.type == "cuda" else "cpu",
logger=False,
enable_checkpointing=False,
enable_model_summary=False,
gradient_clip_val=1.0,
)
trainer.fit(Reconstruction(model), train_loader, valid_loader)

model.to(DEVICE)
print(f"rho ended at {float(block.rho.detach()):.3f}")

# %%
Expand Down Expand Up @@ -454,7 +461,7 @@ def quality(estimate):
print(f"{name:>16} PSNR {made[0]:5.2f} dB SSIM {made[1]:.3f}")

# sphinx_gallery_start_ignore
reference = truth[0]
reference = truth[0].cpu()
top = float(reference.abs().max())
learned_name = f"MoDL, K={ITERATIONS}"
zoom = (slice(70, 118), slice(40, 88))
Expand Down
30 changes: 30 additions & 0 deletions src/bartorch/_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,35 @@ def axes_flags(axes, ndim: int) -> int:
return _axes_to_flags(axes, ndim)


def check_device(held: torch.device | None, x: torch.Tensor, what: str) -> None:
"""Raise if ``x`` is on a card and the operator's data is on the host.

BART dereferences the data an operator holds where its operand is, so a
host operator handed a device tensor reads host memory from the card. An
operator built on a card takes host operands: it moves them itself.

Parameters
----------
held : torch.device or None
Where the operator's data is; ``None`` for an operator that holds none.
x : torch.Tensor
The operand.
what : str
The operand's name in the message.

Raises
------
ValueError
When ``x`` is on a card and ``held`` is the host.
"""
if held is None or torch.device(held).type != "cpu" or x.device.type == "cpu":
return
raise ValueError(
f"{what} is on {x.device} but the operator was built on the host; "
f"build it on {x.device} or move {what} to the host with .cpu()"
)


def as_operand(x: Any, shape: Shape, what: str) -> torch.Tensor:
"""A contiguous complex64 tensor of ``shape``, on a device BART can reach."""
if not isinstance(x, torch.Tensor):
Expand Down Expand Up @@ -288,6 +317,7 @@ def _apply(
f"{type(self).__name__} defines neither _create nor this method"
)
x = as_operand(x, ishape, "input")
check_device(self.device, x, "input")
if out is None:
y = torch.empty(oshape, dtype=torch.complex64, device=x.device)
else:
Expand Down
5 changes: 4 additions & 1 deletion src/bartorch/optim/_linear.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from bartorch import _marshal
from bartorch._dispatch import BartError, _ensure_ready, _lock, _on_device
from bartorch._lib import library
from bartorch._operator import as_operand
from bartorch._operator import as_operand, check_device
from bartorch.optim._blocks import (
ADMMBlock,
FISTABlock,
Expand Down Expand Up @@ -335,10 +335,13 @@ def _solve(
appended to it."""
op = A._bart()
y = as_operand(y, op.oshape, "y")
check_device(op.device, y, "y")
if x0 is None:
x = torch.zeros(op.ishape, dtype=torch.complex64, device=y.device)
else:
x = as_operand(x0, op.ishape, "x0").clone()
if x.device != y.device:
raise ValueError(f"x0 is on {x.device} but y is on {y.device}")

_ensure_ready()
lib = library()
Expand Down
46 changes: 46 additions & 0 deletions tests/test_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import bartorch._reference as ref
import bartorch.tools as bt
from bartorch import linop
from bartorch._operator import check_device

requires_cuda = pytest.mark.skipif(
not bartorch._cuda.available(), reason="no CUDA device, or the library was built without CUDA"
Expand Down Expand Up @@ -169,3 +170,48 @@ def test_a_tool_that_is_not_kept_on_the_card_still_answers_on_it():
assert out.device.type == "cuda"
expected = bartorch.fftmod(img.cpu(), (-2, -1))
torch.testing.assert_close(out.cpu(), expected, rtol=1e-4, atol=1e-5)


def _sense_on(device: str, n: int = 16, coils: int = 2) -> linop.CartesianSense:
sensitivities = torch.ones(coils, n, n, dtype=torch.complex64, device=device)
pattern = torch.ones(n, n, dtype=torch.complex64, device=device)
return linop.CartesianSense(sensitivities, (n, n), pattern=pattern)


def test_a_device_operand_for_a_host_operator_is_refused_before_bart_sees_it():
x = torch.empty(4, dtype=torch.complex64, device="meta")
with pytest.raises(ValueError, match="built on the host"):
check_device(torch.device("cpu"), x, "input")


def test_an_operator_holding_no_data_takes_an_operand_anywhere():
check_device(None, torch.empty(4, dtype=torch.complex64, device="meta"), "input")


@requires_cuda
def test_a_device_operand_for_a_host_operator_raises_instead_of_crashing():
A = _sense_on("cpu")
x = torch.randn(A.ishape, dtype=torch.complex64, device="cuda")
y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda")
with pytest.raises(ValueError, match="built on the host"):
A(x)
with pytest.raises(ValueError, match="built on the host"):
A.H(y)
with pytest.raises(ValueError, match="built on the host"):
bartorch.optim.cg(y, A, maxiter=2)


@requires_cuda
def test_a_starting_image_made_on_the_bare_cuda_device_is_on_the_current_card():
A = _sense_on("cuda")
y = torch.randn(A.oshape, dtype=torch.complex64, device=f"cuda:{torch.cuda.current_device()}")
x0 = torch.zeros(A.ishape, dtype=torch.complex64, device="cuda")
assert bartorch.optim.cg(y, A, x0=x0, maxiter=2).device == y.device


@requires_cuda
def test_a_starting_image_on_another_device_than_the_data_is_refused():
A = _sense_on("cuda")
y = torch.randn(A.oshape, dtype=torch.complex64, device="cuda")
with pytest.raises(ValueError, match="x0 is on cpu"):
bartorch.optim.cg(y, A, x0=torch.zeros(A.ishape, dtype=torch.complex64), maxiter=2)
Loading