From 22981f327d8244aed00f60026f24a950f559a849 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:27:49 +0000 Subject: [PATCH 1/8] Real-subspace kernels write the final complex signal, receiver phase and sample sign folded in The real kernels store each sample as an interleaved complex64 multiplied by a per-sample complex factor, so the forward no longer makes a torch.complex pass and a full-signal multiply after the kernel. A simulator's constant receiver phase (1j for MRF, MPRAGE, MPnRAGE) and the sign the packing moves off the pulses reach the kernel as that factor. The turned flip and phase buffers are remembered per stream, so the pulse-axis check is read once rather than on every call. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- src/blochsim/_epg_kernels.hpp | 22 ++- src/blochsim/_kernels.hpp | 4 +- src/blochsim/_layout.cu | 4 +- src/blochsim/_layout_real.hpp | 12 +- src/blochsim/_model/_state_machine.py | 4 + src/blochsim/sequence/_accelerators.py | 81 +++++++-- src/blochsim/sequence/_epg_gpu.py | 243 ++++++++++++++++--------- src/blochsim/sequence/_simulation.py | 7 +- src/blochsim/simulators/mpnrage.py | 7 +- src/blochsim/simulators/mprage.py | 3 +- src/blochsim/simulators/mrf.py | 3 +- 11 files changed, 267 insertions(+), 123 deletions(-) diff --git a/src/blochsim/_epg_kernels.hpp b/src/blochsim/_epg_kernels.hpp index e3f434d9..be28c4b1 100644 --- a/src/blochsim/_epg_kernels.hpp +++ b/src/blochsim/_epg_kernels.hpp @@ -9154,7 +9154,7 @@ BSK_HD void _epg_real_vjp_jvp_kernel(float* t1, float* t2, float* m0, float* b1, bsk::atomic_add(((grad_tissue_tangent + ((7 + past_transmit) * atom_count)) + atom), grad_damping_tangent, active_atom); } -BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { +BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { bsk::V alpha{}; bsk::V atom_b1{}; bsk::V atom_damping{}; @@ -9302,8 +9302,13 @@ BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* auto out_ = bsk::ld((output_index + event)); auto output_offset = ((problem * output_count) + out_); auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0)); - bsk::st(((output_real + output_offset) + state), empty, output_mask); - bsk::st(((output_imag + output_offset) + state), (atom_m0 * plus), output_mask); + auto output_at = (output + ((output_offset + state) * 2)); + // The signal is i times the state, before its factor. + auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2))); + auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1)); + auto held = (atom_m0 * plus); + bsk::st(output_at, (-factor_imag * held), output_mask); + bsk::st((output_at + 1), (factor_real * held), output_mask); } if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) { auto t2_ = _shift_real(plus, minus, state, state_mask, state_count); @@ -12008,7 +12013,7 @@ BSK_HD void _epg_vjp_kernel(float* t1, float* t2, float* m0, float* b1, float* b }); } -BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { +BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { bsk::V atom_b1{}; bsk::V atom_damping{}; bsk::V atom_inversion{}; @@ -12201,8 +12206,13 @@ BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, flo auto output_offset = ((problem * output_count) + out_); auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0)); auto signal_imag = ((dot_m0 * plus) + (atom_m0 * dot_plus)); - bsk::st(((output_real + output_offset) + state), empty, output_mask); - bsk::st(((output_imag + output_offset) + state), signal_imag, output_mask); + auto output_at = (output + ((output_offset + state) * 2)); + // The signal is i times the state, before its factor. + auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2))); + auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1)); + auto held = signal_imag; + bsk::st(output_at, (-factor_imag * held), output_mask); + bsk::st((output_at + 1), (factor_real * held), output_mask); } if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) { auto t4_ = _shift_real(plus, minus, state, state_mask, state_count); diff --git a/src/blochsim/_kernels.hpp b/src/blochsim/_kernels.hpp index 74e79831..3e7df8d9 100644 --- a/src/blochsim/_kernels.hpp +++ b/src/blochsim/_kernels.hpp @@ -818,8 +818,8 @@ inline constexpr KernelInfo KERNELS[] = { {"_epg_vjp_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,grad_pair_value,grad_pair_tangent,dot_t1,dot_t2,dot_m0,dot_b1,dot_b1_phase,dot_b0,dot_inversion_efficiency,dot_diffusion,dot_velocity,dot_bound_fraction,dot_exchange_rate,dot_t1_bound,dot_pool_b_fraction,dot_pool_b_exchange,dot_t1_pool_b,dot_t2_pool_b,dot_pool_b_shift,dot_duration,dot_flip,dot_phase,duration_row,pool_table,pool_bars,pool_durations,row_count,grad_output_real,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_phase_value,grad_phase_tangent,grad_duration_value,grad_duration_tangent,trajectory_vr,trajectory_vi,trajectory_tr,trajectory_ti,problem_base,problem_end,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,directed,off_axis,moving,diffusing,transmit,density,inverting,broadened,lineshape_bins,pools,narrow,tabulated,recording,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppippppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiiii", 105, 106, -1, BLOCHSIM_LANES__epg_vjp_jvp_kernel}, {"_epg_real_vjp_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,dot_t1,dot_t2,dot_m0,dot_b1,dot_inversion_efficiency,dot_diffusion,dot_duration,dot_flip,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_duration_value,grad_duration_tangent,trajectory_value,trajectory_tangent,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppiiiiiiiiiiiiiiiii", 44, 45, -1, BLOCHSIM_LANES__epg_real_vjp_jvp_kernel}, {"_epg_real_vjp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,grad_output_imag,grad_tissue,grad_flip,grad_duration,trajectory_value,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppiiiiiiiiiiiiiiiii", 32, 33, -1, BLOCHSIM_LANES__epg_real_vjp_kernel}, - {"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel}, - {"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel}, + {"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel}, + {"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel}, {"_epg_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,bound_exchange,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,phase_cos,phase_sin,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 65, 66, -1, BLOCHSIM_LANES__epg_kernel}, {"_epg_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_b1_phase,tangent_b0,tangent_inversion_efficiency,tangent_diffusion,tangent_velocity,tangent_bound_fraction,tangent_exchange_rate,tangent_t1_bound,tangent_pool_b_fraction,tangent_pool_b_exchange,tangent_t1_pool_b,tangent_t2_pool_b,tangent_pool_b_shift,tangent_duration,tangent_flip,tangent_phase,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 84, 85, -1, BLOCHSIM_LANES__epg_jvp_kernel}, {"_pooled_kernel", "m0,b1,b1_phase,b0,efficiency,diffusion,velocity,dm0,db1,db1_phase,db0,defficiency,ddiffusion,dvelocity,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,dduration,dflip,dphase,table,dtable,pool_index,profile,profile_index,lineshape,pairs,pair_index,dpairs,output_real,output_imag,trajectory,base,atom_count,event_count,output_count,state_count,rows,flow_scale,washout_scale,profile_step,lineshape_step,locations,profile_bins,lineshape_bins,n,m,blocks,planes,atom_stride,shimmed,profiled,dynamic,directed_pairs,directed_table,following,keep,off_axis,moving,diffusing,transmit,density,inverting,P,S", "ppppppppppppppppppppppppppppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiii", 70, 69, 69, BLOCHSIM_LANES__pooled_kernel}, diff --git a/src/blochsim/_layout.cu b/src/blochsim/_layout.cu index 9f918e4a..6c541df5 100644 --- a/src/blochsim/_layout.cu +++ b/src/blochsim/_layout.cu @@ -140,8 +140,8 @@ layout_real::Params real_params(const Reader& r) { p.output_index = r.ints("output_index"); p.shim_index = r.ints("shim_index"); p.action = static_cast(r.a[r.at("action")].p); - p.output_real = r.outputs("output_real"); - p.output_imag = r.outputs("output_imag"); + p.output = r.outputs("output"); + p.sample_factor = r.floats("sample_factor"); p.atom_count = static_cast(r.integer("atom_count")); p.train_count = static_cast(r.integer("train_count")); p.event_count = static_cast(r.integer("event_count")); diff --git a/src/blochsim/_layout_real.hpp b/src/blochsim/_layout_real.hpp index 73fbb16f..b2e68798 100644 --- a/src/blochsim/_layout_real.hpp +++ b/src/blochsim/_layout_real.hpp @@ -23,7 +23,10 @@ struct Params { const float *d_t1, *d_t2, *d_m0, *d_b1, *d_inversion_efficiency, *d_diffusion, *d_duration, *d_flip; const int *kind, *output_index, *shim_index; const unsigned char* action; - float *output_real, *output_imag; + // Each sample as a complex number, real then imaginary; and the complex + // factor each sample is multiplied by, or null for none. + float* output; + const float* sample_factor; int atom_count, train_count, event_count, output_count, state_count, width; bool single_train, atom_stride, shimmed, diffusing, transmit, density, inverting; }; @@ -123,12 +126,15 @@ __device__ __forceinline__ void real_loop(const Params& p) { float held[Y]; int run_start = 0, run_len = 0; auto flush = [&]() { + // The signal is i times what a thread holds, before its factor. + const float2 factor = p.sample_factor && state < run_len + ? reinterpret_cast(p.sample_factor)[run_start + state] + : make_float2(1.0f, 0.0f); #pragma unroll for (int y = 0; y < Y; ++y) { if (state < run_len && active[y]) { const long at_out = static_cast(problem[y]) * p.output_count + run_start + state; - p.output_real[at_out] = 0.0f; - p.output_imag[at_out] = held[y]; + reinterpret_cast(p.output)[at_out] = make_float2(-factor.y * held[y], factor.x * held[y]); } } run_len = 0; diff --git a/src/blochsim/_model/_state_machine.py b/src/blochsim/_model/_state_machine.py index a3090e81..466c889d 100644 --- a/src/blochsim/_model/_state_machine.py +++ b/src/blochsim/_model/_state_machine.py @@ -529,6 +529,9 @@ class Simulator(_SignalModel): across_slice: Any = None crusher_dephasing_rad: float = 0.0 voxel_size_m: float | None = None + # The constant every recorded sample is multiplied by: the receiver phase + # that puts a sequence's signal on the axis its readers expect. + receiver: complex = 1.0 # Every attribute the constructor sets has a value here as well, so a # closed form that writes an __init__ of its own -- taking the two blocks @@ -1006,6 +1009,7 @@ def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tens nstates=states, events=events, across_slice=profile, + _receiver=self.receiver, **settings, ) .signal diff --git a/src/blochsim/sequence/_accelerators.py b/src/blochsim/sequence/_accelerators.py index 17f9933f..a5bf6fb7 100644 --- a/src/blochsim/sequence/_accelerators.py +++ b/src/blochsim/sequence/_accelerators.py @@ -601,14 +601,28 @@ def _one_axis(packed: _PackedEvents) -> tuple[_PackedEvents, torch.Tensor | None if corrections is None: return packed, None out_by, flips, samples = corrections - return ( - replace( - packed, - flip=(packed.flip * flips).contiguous(), - phase=(packed.phase - out_by * torch.pi).contiguous(), - ), - samples, + + def turned() -> tuple[torch.Tensor, torch.Tensor]: + return ( + (packed.flip * flips).contiguous(), + (packed.phase - out_by * torch.pi).contiguous(), + ) + + # The turned buffers are kept with the stream they came from, so what is + # remembered against them downstream -- the pulse axis above all, which is + # a device round trip -- is read once per stream rather than once per call. + # A buffer carrying a graph is turned afresh, so no call backpropagates + # through another's. + flip, phase = ( + turned() + if packed.flip.requires_grad or packed.phase.requires_grad + else _remembered( + "turned onto one axis", + (packed.kind, packed.flip, packed.phase, packed.action), + turned, + ) ) + return replace(packed, flip=flip, phase=phase), samples def simulate_native( @@ -630,6 +644,7 @@ def simulate_native( record_every: bool = False, pools: int = 0, extra_pools: tuple[torch.Tensor, ...] | None = None, + receiver: complex = 1.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] | None: """Run a fused CPU/CUDA state machine with explicit AD rules. @@ -651,6 +666,8 @@ def simulate_native( what is skipped is walking the events again to rebuild what only their values changed. + ``receiver`` is the constant every recorded sample is multiplied by. + ``pools`` counts the chemically exchanging pools the tissue carries from pool B on, and ``extra_pools`` holds the prepared buffers of pools C to E. From :data:`~blochsim.sequence._pools.TABULATE_FROM` pools on, the kernels @@ -766,6 +783,7 @@ def simulate_native( features, pool_values, pool_index, + _sample_factor(samples, receiver, packed.output_count, device), ) leading = (packed.train_count,) if packed.is_batched else () if locations > 1: @@ -775,8 +793,6 @@ def simulate_native( signal = signal.reshape(*leading, *output_shape, packed.output_count) if dephasing is not None: signal = signal * dephasing - if samples is not None: - signal = signal * samples return ( signal, packed.time_us, @@ -1491,6 +1507,7 @@ def forward( features: frozenset[str] | None, pool_values: torch.Tensor | None = None, pool_index: torch.Tensor | None = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: tissue = ( t1, @@ -1535,6 +1552,7 @@ def forward( exchanging=exchanging, features=features, pools=pools_of(pool_values, pool_index, exchanging), + factor=factor, ) @staticmethod @@ -1559,6 +1577,9 @@ def setup_context(ctx: Any, inputs: tuple[Any, ...], _output: torch.Tensor) -> N ctx.pool_index = ( inputs[_PACKED_COUNT + 11] if len(inputs) > _PACKED_COUNT + 11 else None ) + ctx.factor = ( + inputs[_PACKED_COUNT + 12] if len(inputs) > _PACKED_COUNT + 12 else None + ) @staticmethod @_under_recorded_policy @@ -1570,7 +1591,7 @@ def jvp(ctx: Any, *tangents: torch.Tensor | None) -> torch.Tensor: else tangents[index].to(saved[index]).contiguous() for index in _FLOAT_INPUTS ) - return _NativeEpgJvp.apply( + tangent = _NativeEpgJvp.apply( *saved, *float_tangents, ctx.state_count, @@ -1593,12 +1614,15 @@ def jvp(ctx: Any, *tangents: torch.Tensor | None) -> torch.Tensor: ctx.pool_values, ), ) + return tangent if ctx.factor is None else tangent * ctx.factor @staticmethod @_under_recorded_policy def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[Any, ...]: saved = ctx.saved_tensors wanted = _wanted(ctx.needs_input_grad) + if ctx.factor is not None: + grad_output = grad_output * ctx.factor.conj() fused = _NativeEpgVjp.apply( *saved, grad_output, @@ -1637,6 +1661,7 @@ def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[Any, ...]: None, pool_grad if _needs(ctx, _PACKED_COUNT + 10) else None, None, + None, ) @staticmethod @@ -3047,6 +3072,7 @@ def _run_packed( dynamic: Any = None, features: frozenset[str] | None = None, pools: PoolTables | None = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: """Run the forward state machine. @@ -3060,6 +3086,9 @@ def _run_packed( ``features`` names the terms the tissue gives anything to do, and the kernels carry those and no others. ``None`` leaves every term in. + + ``factor`` is a complex number per output sample that multiplies the + signal. """ profile = _tables(profile, events, dynamic) if real_axis is None: @@ -3108,6 +3137,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=piece["dynamic"], features=features, pools=piece["pools"], + factor=_on(factor, device), ), ) from ._epg_gpu import simulate @@ -3126,6 +3156,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=piece["dynamic"], features=features, pools=piece["pools"], + factor=_on(factor, device), ), ) @@ -3163,6 +3194,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: lineshape=lineshape, exchanging=exchanging, features=features, + factor=_on(factor, device), ), ) for begin, end, device in shards @@ -3181,6 +3213,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=dynamic, features=features, pools=pools, + factor=factor, ) from blochsim import _epg_cpu @@ -3225,7 +3258,33 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: feature_mask(features, geometry), *_pool_counts(pools), ) - return torch.complex(output_real, output_imag) + signal = torch.complex(output_real, output_imag) + return signal if factor is None else signal * factor + + +def _sample_factor( + samples: torch.Tensor | None, + receiver: complex, + output_count: int, + device: torch.device, +) -> torch.Tensor | None: + """The complex number each recorded sample is multiplied by, or ``None``. + + ``samples`` is the sign the packing moved off the pulses, one per sample, + and ``receiver`` the constant the caller asked for. + """ + if samples is None and receiver == 1: + return None + if samples is None: + return torch.full( + (output_count,), complex(receiver), dtype=torch.complex64, device=device + ) + return (samples * complex(receiver)).to(torch.complex64).contiguous() + + +def _on(value: torch.Tensor | None, device: torch.device) -> torch.Tensor | None: + """``value`` on ``device``, or ``None``.""" + return None if value is None else value.to(device) def _card_vjp( diff --git a/src/blochsim/sequence/_epg_gpu.py b/src/blochsim/sequence/_epg_gpu.py index eb84c3a1..49eca722 100644 --- a/src/blochsim/sequence/_epg_gpu.py +++ b/src/blochsim/sequence/_epg_gpu.py @@ -364,16 +364,20 @@ def simulate( dynamic: Any = None, features: frozenset[str] | None = None, pools: Any = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: """Run a packed state machine on CUDA and return complex signals. ``real_axis`` of 1 selects the real-subspace kernel; see ``real_subspace_axis`` for when that is legitimate. + + ``factor`` is a complex number per output sample, ``(output_count,)``, + that multiplies the signal; ``None`` leaves it as the kernels produce it. """ if pools is not None: from . import _pools_gpu - return _pools_gpu.simulate( + signal = _pools_gpu.simulate( tissue, events, state_count=state_count, @@ -385,13 +389,25 @@ def simulate( features=features, pools=pools, ) + return signal if factor is None else signal * factor train_count = _train_count(events) atom_count = tissue[0].numel() - output_real = torch.empty( - _output_shape(train_count, atom_count, output_count), - dtype=torch.float32, - device=tissue[0].device, - ) + shape = _output_shape(train_count, atom_count, output_count) + if real_axis == 1: + signal = torch.empty(shape, dtype=torch.complex64, device=tissue[0].device) + _simulate_real( + tissue, + events, + signal, + state_count=state_count, + output_count=output_count, + atom_count=atom_count, + geometry=geometry, + features=features, + factor=factor, + ) + return signal + output_real = torch.empty(shape, dtype=torch.float32, device=tissue[0].device) output_imag = torch.empty_like(output_real) simulate_into( tissue, @@ -400,7 +416,6 @@ def simulate( output_imag, state_count=state_count, output_count=output_count, - real_axis=real_axis, atom_count=atom_count, geometry=geometry, profile=profile, @@ -409,7 +424,60 @@ def simulate( dynamic=dynamic, features=features, ) - return torch.complex(output_real, output_imag) + signal = torch.complex(output_real, output_imag) + return signal if factor is None else signal * factor + + +def _simulate_real( + tissue: tuple[torch.Tensor, ...], + events: tuple[torch.Tensor, ...], + signal: torch.Tensor, + *, + state_count: int, + output_count: int, + atom_count: int, + geometry: Geometry, + features: frozenset[str] | None, + factor: torch.Tensor | None, +) -> None: + """Run the real-subspace kernel into a complex signal the caller owns. + + The kernel writes each sample as a complex number with its ``factor`` + already applied. + """ + t1, t2, m0, b1, _, _, inversion_efficiency, diffusion = tissue[:8] + duration, kind, flip, _, action, output_index, shim_index = events[:7] + train_count = _train_count(events) + block_states = next_power_of_2(state_count) + problems = _problems_per_program(block_states, _epg_real_kernel) + grid = (cdiv(train_count * atom_count, problems),) + _epg_real_kernel[grid]( + t1, + t2, + m0, + b1, + inversion_efficiency, + diffusion, + duration, + kind, + flip, + action, + output_index, + shim_index, + torch.view_as_real(signal), + None if factor is None else torch.view_as_real(factor.to(torch.complex64)), + atom_count, + train_count, + kind.numel(), + output_count, + state_count=state_count, + single_train=train_count == 1, + atom_stride=_atom_stride(tissue), + shimmed=_shim_count(tissue) > 1, + **_only_scalars(_feature_flags(features, geometry)), + block_states=block_states, + problems=problems, + ) def simulate_into( @@ -420,7 +488,6 @@ def simulate_into( *, state_count: int, output_count: int, - real_axis: int | None, atom_count: int, geometry: Geometry = NO_GEOMETRY, profile: Any = None, @@ -473,9 +540,7 @@ def simulate_into( pools = _pool_flag(lineshape, exchanging) block_states = next_power_of_2(state_count) total = train_count * atom_count - problems = _problems_per_program( - block_states, _epg_real_kernel if real_axis == 1 else _epg_kernel - ) + problems = _problems_per_program(block_states, _epg_kernel) grid = (cdiv(total, problems),) # A kernel argument has to be a tensor even where the branch reading it is # compiled out, so an unprofiled launch passes one it already has. @@ -498,36 +563,6 @@ def simulate_into( phase_cos = torch.cos(phase.double()).to(torch.float32) phase_sin = torch.sin(phase.double()).to(torch.float32) - if real_axis == 1: - _epg_real_kernel[grid]( - t1, - t2, - m0, - b1, - inversion_efficiency, - diffusion, - duration, - kind, - flip, - action, - output_index, - shim_index, - output_real, - output_imag, - atom_count, - train_count, - kind.numel(), - output_count, - state_count=state_count, - single_train=train_count == 1, - atom_stride=_atom_stride(tissue), - shimmed=_shim_count(tissue) > 1, - **_only_scalars(_feature_flags(features, geometry)), - block_states=block_states, - problems=problems, - ) - return - _epg_kernel[grid]( t1, t2, @@ -638,11 +673,23 @@ def simulate_jvp( ) train_count = _train_count(events) atom_count = tissue[0].numel() - output_real = torch.empty( - _output_shape(train_count, atom_count, output_count), - dtype=torch.float32, - device=tissue[0].device, - ) + shape = _output_shape(train_count, atom_count, output_count) + if real_axis == 1: + tangent = torch.empty(shape, dtype=torch.complex64, device=tissue[0].device) + _simulate_real_jvp( + tissue, + events, + tissue_tangents, + event_tangents, + tangent, + state_count=state_count, + output_count=output_count, + atom_count=atom_count, + geometry=geometry, + features=features, + ) + return tangent + output_real = torch.empty(shape, dtype=torch.float32, device=tissue[0].device) output_imag = torch.empty_like(output_real) simulate_jvp_into( tissue, @@ -653,7 +700,6 @@ def simulate_jvp( output_imag, state_count=state_count, output_count=output_count, - real_axis=real_axis, atom_count=atom_count, geometry=geometry, profile=profile, @@ -666,6 +712,64 @@ def simulate_jvp( return torch.complex(output_real, output_imag) +def _simulate_real_jvp( + tissue: tuple[torch.Tensor, ...], + events: tuple[torch.Tensor, ...], + tissue_tangents: tuple[torch.Tensor, ...], + event_tangents: tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tangent: torch.Tensor, + *, + state_count: int, + output_count: int, + atom_count: int, + geometry: Geometry, + features: frozenset[str] | None, +) -> None: + """Run the real-subspace Jacobian-vector product into a complex tangent.""" + t1, t2, m0, b1, _, _, inversion_efficiency, diffusion = tissue[:8] + duration, kind, flip, _, action, output_index, shim_index = events[:7] + tangent_duration, tangent_flip, _ = event_tangents + train_count = _train_count(events) + block_states = next_power_of_2(state_count) + problems = _problems_per_program(block_states, _epg_real_jvp_kernel) + grid = (cdiv(train_count * atom_count, problems),) + _epg_real_jvp_kernel[grid]( + t1, + t2, + m0, + b1, + inversion_efficiency, + diffusion, + duration, + kind, + flip, + action, + output_index, + shim_index, + tissue_tangents[0], + tissue_tangents[1], + tissue_tangents[2], + tissue_tangents[3], + tissue_tangents[6], + tissue_tangents[7], + tangent_duration, + tangent_flip, + torch.view_as_real(tangent), + None, + atom_count, + train_count, + kind.numel(), + output_count, + state_count=state_count, + single_train=train_count == 1, + atom_stride=_atom_stride(tissue, tissue_tangents), + shimmed=_shim_count(tissue) > 1, + **_only_scalars(_feature_flags(features, geometry)), + block_states=block_states, + problems=problems, + ) + + def simulate_jvp_into( tissue: tuple[torch.Tensor, ...], events: tuple[torch.Tensor, ...], @@ -676,7 +780,6 @@ def simulate_jvp_into( *, state_count: int, output_count: int, - real_axis: int | None, atom_count: int, geometry: Geometry = NO_GEOMETRY, profile: Any = None, @@ -726,49 +829,9 @@ def simulate_jvp_into( shims = _shim_count(tissue) block_states = next_power_of_2(state_count) total = train_count * atom_count - problems = _problems_per_program( - block_states, _epg_real_jvp_kernel if real_axis == 1 else _epg_jvp_kernel - ) + problems = _problems_per_program(block_states, _epg_jvp_kernel) grid = (cdiv(total, problems),) - if real_axis == 1: - _epg_real_jvp_kernel[grid]( - t1, - t2, - m0, - b1, - inversion_efficiency, - diffusion, - duration, - kind, - flip, - action, - output_index, - shim_index, - tissue_tangents[0], - tissue_tangents[1], - tissue_tangents[2], - tissue_tangents[3], - tissue_tangents[6], - tissue_tangents[7], - tangent_duration, - tangent_flip, - output_real, - output_imag, - atom_count, - train_count, - kind.numel(), - output_count, - state_count=state_count, - single_train=train_count == 1, - atom_stride=_atom_stride(tissue, tissue_tangents), - shimmed=shims > 1, - **_only_scalars(_feature_flags(features, geometry)), - block_states=block_states, - problems=problems, - ) - return - table = None if profile is None else profile.packed(t1.device) pairs = None if dynamic is None else dynamic.packed(t1.device) pair_rows = ( diff --git a/src/blochsim/sequence/_simulation.py b/src/blochsim/sequence/_simulation.py index 6fb96bce..1fda6fdb 100644 --- a/src/blochsim/sequence/_simulation.py +++ b/src/blochsim/sequence/_simulation.py @@ -374,6 +374,7 @@ def simulate( device: torch.device | str | None = None, events: Any = None, _every: bool = False, + _receiver: complex = 1.0, ) -> SimulationResult: """Walk an event stream and record selected ADC signals. @@ -400,7 +401,7 @@ def simulate( if rf_raster_time_s is None: rf_raster_time_s = description.rf_raster_time_s if repetitions == AUTO: - return self._settled( + settled = self._settled( description, tissue, record=record, @@ -409,6 +410,9 @@ def simulate( rf_raster_time_s=rf_raster_time_s, device=device, ) + if _receiver == 1: + return settled + return replace(settled, signal=settled.signal * _receiver) repetitions = _as_integer(repetitions, "repetitions") if repetitions < 1: raise ValueError("repetitions must be positive") @@ -487,6 +491,7 @@ def simulate( record_every=_every, pools=pools, extra_pools=extra_pools, + receiver=_receiver, ) if accelerated is None: raise RuntimeError( diff --git a/src/blochsim/simulators/mpnrage.py b/src/blochsim/simulators/mpnrage.py index 6eef7f74..157cbeac 100644 --- a/src/blochsim/simulators/mpnrage.py +++ b/src/blochsim/simulators/mpnrage.py @@ -4,8 +4,6 @@ __all__ = ["MPnRAGESimulator"] -from collections.abc import Mapping -from typing import Any import numpy.typing as npt import torch @@ -71,6 +69,7 @@ class MPnRAGESimulator(Simulator): operators=SPOILED, ) states = 1 + receiver = 1j def layout( self, @@ -107,7 +106,3 @@ def layout( self.operators.readout(turns[index], duration_s=spacing_s[index]) ) return parts - - def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tensor: - """Return the signal at every shot.""" - return 1j * super().evaluate(properties, **sequence) diff --git a/src/blochsim/simulators/mprage.py b/src/blochsim/simulators/mprage.py index 2b3d522d..3ad18de8 100644 --- a/src/blochsim/simulators/mprage.py +++ b/src/blochsim/simulators/mprage.py @@ -74,6 +74,7 @@ class MPRAGESimulator(Simulator): ) states = 1 record = "acquired" + receiver = 1j def layout( self, @@ -125,7 +126,7 @@ def layout( def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tensor: """Return the acquired sample.""" - return 1j * super().evaluate(properties, **sequence)[..., 0] + return super().evaluate(properties, **sequence)[..., 0] # %% private module subroutines diff --git a/src/blochsim/simulators/mrf.py b/src/blochsim/simulators/mrf.py index 95433099..6ede62f1 100644 --- a/src/blochsim/simulators/mrf.py +++ b/src/blochsim/simulators/mrf.py @@ -74,6 +74,7 @@ class MRFSimulator(Simulator): operators=UNBALANCED, ) states = 10 + receiver = 1j def layout( self, @@ -101,4 +102,4 @@ def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tens signal = super().evaluate(properties, **sequence) played = self.played(**sequence) echoes = torch.atleast_1d(played["flip"]).shape[-1] - return 1j * signal[..., -echoes:] + return signal[..., -echoes:] From 328673e8cd5ff5ffb41a9b0b3cd801113660942f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:27:49 +0000 Subject: [PATCH 2/8] TEMP: kernel variant sweep for the A40 (dropped before merge) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- benchmarks/sweep/_layout_real.cu | 58 ++++++++++++++++++++ benchmarks/sweep_real.py | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 benchmarks/sweep/_layout_real.cu create mode 100644 benchmarks/sweep_real.py diff --git a/benchmarks/sweep/_layout_real.cu b/benchmarks/sweep/_layout_real.cu new file mode 100644 index 00000000..6d56088d --- /dev/null +++ b/benchmarks/sweep/_layout_real.cu @@ -0,0 +1,58 @@ +// EXPERIMENT ONLY: the real kernel and its JVP instantiated over a grid of +// (Y, warps per block, minimum blocks per SM), picked at run time from +// BLOCHSIM_REAL_SWEEP="Y,W,OCC" (forward) and BLOCHSIM_REAL_JVP_SWEEP (dual), +// where OCC is the resident-warp target per SM (0 = no bound). +#include +#include +#include "_layout.hpp" + +namespace blochsim_layout { +namespace { + +template +__global__ void __launch_bounds__(32 * W, MINB) sweep_kernel(layout_real::Params p) { + layout_real::real_loop(p); +} + +template +int go(const layout_real::Params& p, cudaStream_t stream) { + const int groups = 32 / p.width; + const long long problems = static_cast(p.train_count) * p.atom_count; + const long long per_block = static_cast(W) * groups * Y; + const unsigned grid = static_cast((problems + per_block - 1) / per_block); + if (!p.single_train || p.transmit || p.density || p.inverting) return 1; + sweep_kernel<<>>(p); + return static_cast(cudaGetLastError()); +} + +#define OCCS(T, Y, W) \ + if (y == Y && w == W && occ == 0) return go(p, stream); \ + if (y == Y && w == W && occ == 16) return go(p, stream); \ + if (y == Y && w == W && occ == 24) return go(p, stream); \ + if (y == Y && w == W && occ == 32) return go(p, stream); +#define WS(T, Y) OCCS(T, Y, 2) OCCS(T, Y, 4) OCCS(T, Y, 8) + +template +int dispatch(const char* env, int y_default, bool capped, const layout_real::Params& p, cudaStream_t stream) { + int y = y_default, w = WARPS, occ = capped ? 16 : 0; + if (const char* s = std::getenv(env)) std::sscanf(s, "%d,%d,%d", &y, &w, &occ); + if constexpr (num::is_dual::value) { + WS(T, 1) WS(T, 2) WS(T, 4) + } else { + WS(T, 2) WS(T, 4) WS(T, 8) + } + std::fprintf(stderr, "blochsim sweep: no variant %d,%d,%d\n", y, w, occ); + return 1; +} + +} // namespace + +int real_forward(const layout_real::Params& p, cudaStream_t stream) { + return dispatch("BLOCHSIM_REAL_SWEEP", 8, false, p, stream); +} + +int real_jvp(const layout_real::Params& p, cudaStream_t stream) { + return dispatch("BLOCHSIM_REAL_JVP_SWEEP", 4, true, p, stream); +} + +} // namespace blochsim_layout diff --git a/benchmarks/sweep_real.py b/benchmarks/sweep_real.py new file mode 100644 index 00000000..660bfecb --- /dev/null +++ b/benchmarks/sweep_real.py @@ -0,0 +1,94 @@ +"""Time MRF-FISP (500 TRs, 32 states) forward and T1/T2 Jacobian per kernel variant. + +Run from the blochsim-bench env with a scratch build of the sweep branch: + python sweep.py --atoms 100000 [--profile] +""" + +import argparse +import itertools +import os +import sys + +import torch + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + + +def main(): + cli = argparse.ArgumentParser() + cli.add_argument("--atoms", type=int, default=100_000) + cli.add_argument("--length", type=int, default=500) + cli.add_argument("--states", type=int, default=32) + cli.add_argument("--repeats", type=int, default=30) + cli.add_argument("--profile", action="store_true") + cli.add_argument("--no-sweep", action="store_true") + args = cli.parse_args() + + from blochsim.simulators import MRFSimulator + + device = torch.device("cuda") + from _common import flip_train, tissue_grid + + flip = torch.tensor(flip_train(args.length), dtype=torch.float32, device=device) + t1, t2 = tissue_grid(args.atoms) + T1 = torch.tensor(t1, dtype=torch.float32, device=device) + T2 = torch.tensor(t2, dtype=torch.float32, device=device) + seq = MRFSimulator(flip=flip, TR=10.0, states=args.states).to(device) + seq.simulate(T1=T1[:1], T2=T2[:1]) + seq.jacobian(("T1", "T2"), T1=T1[:1], T2=T2[:1]) + + def timed(fn, repeats=args.repeats): + for _ in range(3): + fn() + torch.cuda.synchronize() + times = [] + for _ in range(repeats): + a = torch.cuda.Event(enable_timing=True) + b = torch.cuda.Event(enable_timing=True) + a.record() + fn() + b.record() + torch.cuda.synchronize() + times.append(a.elapsed_time(b)) + times.sort() + return times[len(times) // 2] + + fwd = lambda: seq.simulate(T1=T1, T2=T2) # noqa: E731 + jac = lambda: seq.jacobian(("T1", "T2"), T1=T1, T2=T2) # noqa: E731 + reference = fwd() + print(f"forward default: {timed(fwd):.2f} ms", flush=True) + print(f"jacobian default: {timed(jac, 10):.2f} ms", flush=True) + print(f"checksum {complex(reference.sum().cpu())}", flush=True) + + if args.profile: + from torch.profiler import ProfilerActivity, profile + + for name, fn in (("forward", fwd), ("jacobian", jac)): + with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof: + for _ in range(5): + fn() + torch.cuda.synchronize() + print(f"== {name} (5 calls)") + print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=25)) + + if args.no_sweep: + return + for y, w, occ in itertools.product((2, 4, 8), (2, 4, 8), (0, 16, 24, 32)): + os.environ["BLOCHSIM_REAL_SWEEP"] = f"{y},{w},{occ}" + try: + out = fwd() + err = (out - reference).abs().max().item() + print(f"fwd Y={y} W={w} occ={occ}: {timed(fwd):.2f} ms maxerr {err:.1e}", flush=True) + except Exception as e: # noqa: BLE001 + print(f"fwd Y={y} W={w} occ={occ}: failed {e}", flush=True) + os.environ.pop("BLOCHSIM_REAL_SWEEP") + for y, w, occ in itertools.product((1, 2, 4), (2, 4, 8), (0, 16, 24, 32)): + os.environ["BLOCHSIM_REAL_JVP_SWEEP"] = f"{y},{w},{occ}" + try: + print(f"jvp Y={y} W={w} occ={occ}: jacobian {timed(jac, 10):.2f} ms", flush=True) + except Exception as e: # noqa: BLE001 + print(f"jvp Y={y} W={w} occ={occ}: failed {e}", flush=True) + + +if __name__ == "__main__": + main() From 69b9ceed7bf37d8bd46a33de89aace65edf88d3d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:30:17 +0000 Subject: [PATCH 3/8] Build for 8.6 and 8.9; one simulation per Jacobian; fill scalar tissue on the device The Jacobian batches its directions under vmap, so the signal is simulated once and only the directional derivatives are taken per property. Tissue properties left at a plain number are filled on the device rather than copied from the host one by one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- CMakeLists.txt | 9 +++++---- src/blochsim/_model/_signal.py | 25 ++++++++++++++++--------- src/blochsim/sequence/_simulation.py | 5 +++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57099ac7..800911f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,11 +93,12 @@ endif() option(BLOCHSIM_CUDA "Compile the GPU kernels for the card" ${_blochsim_cuda_default}) if(BLOCHSIM_CUDA) - # Machine code for 7.5, 8.0 and 9.0, and PTX for 9.0, which the driver - # compiles for a card newer than any listed. Code for 8.0 runs on every - # 8.x card, so 8.6 and 8.9 take it. + # Machine code for 7.5, 8.0, 8.6, 8.9 and 9.0, and PTX for 9.0, which the + # driver compiles for a card newer than any listed. An 8.6 or 8.9 card + # would run the 8.0 code, but the register allocation nvcc makes for its + # own generation is what the EPG kernels' speed turns on. if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - set(CMAKE_CUDA_ARCHITECTURES 75-real 80-real 90) + set(CMAKE_CUDA_ARCHITECTURES 75-real 80-real 86-real 89-real 90) endif() enable_language(CUDA) find_package(CUDAToolkit REQUIRED) diff --git a/src/blochsim/_model/_signal.py b/src/blochsim/_model/_signal.py index 334fb55b..f51fb413 100644 --- a/src/blochsim/_model/_signal.py +++ b/src/blochsim/_model/_signal.py @@ -229,16 +229,23 @@ def along(*inputs: torch.Tensor) -> torch.Tensor: return self._shaped(self.evaluate(declared, **sequence), batch) forward_mode() - columns = [] - signal = None - for name in names: - tangents = tuple( - torch.ones_like(value) if other == name else torch.zeros_like(value) - for other, value in zip(names, primals, strict=True) + # One direction per property, batched: the signal is simulated once and + # only the directional derivatives are taken per property. + directions = tuple( + torch.stack( + [ + torch.ones_like(value) if other == name else torch.zeros_like(value) + for name in names + ] ) - signal, column = torch.func.jvp(along, primals, tangents) - columns.append(column) - jacobian = columns[0] if isinstance(diff, str) else torch.stack(columns, dim=-2) + for other, value in zip(names, primals, strict=True) + ) + + def column(*tangents: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return torch.func.jvp(along, primals, tangents) + + signal, columns = torch.func.vmap(column, out_dims=(None, 0))(*directions) + jacobian = columns[0] if isinstance(diff, str) else columns.movedim(0, -2) return like(signal, backend), like(jacobian, backend) # -- what a subclass does not have to write ----------------------------- diff --git a/src/blochsim/sequence/_simulation.py b/src/blochsim/sequence/_simulation.py index 1fda6fdb..a3b8e0fc 100644 --- a/src/blochsim/sequence/_simulation.py +++ b/src/blochsim/sequence/_simulation.py @@ -798,6 +798,11 @@ def _prepare_pools( def _as_float_tensor(value: Any, device: torch.device) -> torch.Tensor: if isinstance(value, torch.Tensor): return value.to(device=device, dtype=torch.float32) + if isinstance(value, (int, float)): + # Filled where it lives: a tissue carries some thirty properties left + # at a plain number, and a copy of each from the host would wait on + # the device every call. + return torch.full((), float(value), dtype=torch.float32, device=device) return torch.as_tensor(value, dtype=torch.float32, device=device) From c5ee8ddcb173ccbcf572ee10fa363ca60543afa1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 15:41:32 +0000 Subject: [PATCH 4/8] test: the native forward takes one more trailing argument, the sample factor Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- tests/sequence/test_parameters.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/sequence/test_parameters.py b/tests/sequence/test_parameters.py index 6a7b1b70..4edd70c4 100644 --- a/tests/sequence/test_parameters.py +++ b/tests/sequence/test_parameters.py @@ -202,10 +202,11 @@ def record(needs_input_grad): finally: _accelerators._wanted = original - # The twelve trailing arguments -- state count, output count, thread count, - # the sequence geometry, the transition table, the per-voxel rotations and - # the row each event reads of them, the semisolid pool's lineshape, how the - # chemically exchanging pools are carried, which terms the tissue asks for, - # and the pool tables with the row each event reads of them -- follow the - # packed buffers. - assert widths and all(width == PACKED_COUNT + 12 for width in widths) + # The thirteen trailing arguments -- state count, output count, thread + # count, the sequence geometry, the transition table, the per-voxel + # rotations and the row each event reads of them, the semisolid pool's + # lineshape, how the chemically exchanging pools are carried, which terms + # the tissue asks for, the pool tables with the row each event reads of + # them, and the factor each sample is multiplied by -- follow the packed + # buffers. + assert widths and all(width == PACKED_COUNT + 13 for width in widths) From f33aaaa59596f6846ae12801adffd81c96b0c2ee Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 16:54:17 +0000 Subject: [PATCH 5/8] Cap the real forward kernel at eight programs per SM Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- src/blochsim/_layout_real.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/blochsim/_layout_real.cu b/src/blochsim/_layout_real.cu index ad48a785..cbe83bf7 100644 --- a/src/blochsim/_layout_real.cu +++ b/src/blochsim/_layout_real.cu @@ -4,8 +4,10 @@ namespace blochsim_layout { namespace { +// Held to the registers that let eight programs share an SM; uncapped, nvcc +// spends more and fewer programs hide each other's latency. template -__global__ void __launch_bounds__(32 * WARPS) real_kernel(layout_real::Params p) { +__global__ void __launch_bounds__(32 * WARPS, 8) real_kernel(layout_real::Params p) { layout_real::real_loop(p); } From 5c07572aaf424b7c83b8ef95cbd855819a9892b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 17:07:01 +0000 Subject: [PATCH 6/8] Keep the 7.5, 8.0, 9.0 build targets With the real forward kernel capped, the 8.0 code is the faster one on an 8.6 card, so a separate 8.6 or 8.9 build buys nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 800911f6..57099ac7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,12 +93,11 @@ endif() option(BLOCHSIM_CUDA "Compile the GPU kernels for the card" ${_blochsim_cuda_default}) if(BLOCHSIM_CUDA) - # Machine code for 7.5, 8.0, 8.6, 8.9 and 9.0, and PTX for 9.0, which the - # driver compiles for a card newer than any listed. An 8.6 or 8.9 card - # would run the 8.0 code, but the register allocation nvcc makes for its - # own generation is what the EPG kernels' speed turns on. + # Machine code for 7.5, 8.0 and 9.0, and PTX for 9.0, which the driver + # compiles for a card newer than any listed. Code for 8.0 runs on every + # 8.x card, so 8.6 and 8.9 take it. if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - set(CMAKE_CUDA_ARCHITECTURES 75-real 80-real 86-real 89-real 90) + set(CMAKE_CUDA_ARCHITECTURES 75-real 80-real 90) endif() enable_language(CUDA) find_package(CUDAToolkit REQUIRED) From 09cfcb8612d504b73349e3a18cb2972a27202e68 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 17:07:09 +0000 Subject: [PATCH 7/8] Drop the A40 sweep scaffolding Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- benchmarks/sweep/_layout_real.cu | 58 -------------------- benchmarks/sweep_real.py | 94 -------------------------------- 2 files changed, 152 deletions(-) delete mode 100644 benchmarks/sweep/_layout_real.cu delete mode 100644 benchmarks/sweep_real.py diff --git a/benchmarks/sweep/_layout_real.cu b/benchmarks/sweep/_layout_real.cu deleted file mode 100644 index 6d56088d..00000000 --- a/benchmarks/sweep/_layout_real.cu +++ /dev/null @@ -1,58 +0,0 @@ -// EXPERIMENT ONLY: the real kernel and its JVP instantiated over a grid of -// (Y, warps per block, minimum blocks per SM), picked at run time from -// BLOCHSIM_REAL_SWEEP="Y,W,OCC" (forward) and BLOCHSIM_REAL_JVP_SWEEP (dual), -// where OCC is the resident-warp target per SM (0 = no bound). -#include -#include -#include "_layout.hpp" - -namespace blochsim_layout { -namespace { - -template -__global__ void __launch_bounds__(32 * W, MINB) sweep_kernel(layout_real::Params p) { - layout_real::real_loop(p); -} - -template -int go(const layout_real::Params& p, cudaStream_t stream) { - const int groups = 32 / p.width; - const long long problems = static_cast(p.train_count) * p.atom_count; - const long long per_block = static_cast(W) * groups * Y; - const unsigned grid = static_cast((problems + per_block - 1) / per_block); - if (!p.single_train || p.transmit || p.density || p.inverting) return 1; - sweep_kernel<<>>(p); - return static_cast(cudaGetLastError()); -} - -#define OCCS(T, Y, W) \ - if (y == Y && w == W && occ == 0) return go(p, stream); \ - if (y == Y && w == W && occ == 16) return go(p, stream); \ - if (y == Y && w == W && occ == 24) return go(p, stream); \ - if (y == Y && w == W && occ == 32) return go(p, stream); -#define WS(T, Y) OCCS(T, Y, 2) OCCS(T, Y, 4) OCCS(T, Y, 8) - -template -int dispatch(const char* env, int y_default, bool capped, const layout_real::Params& p, cudaStream_t stream) { - int y = y_default, w = WARPS, occ = capped ? 16 : 0; - if (const char* s = std::getenv(env)) std::sscanf(s, "%d,%d,%d", &y, &w, &occ); - if constexpr (num::is_dual::value) { - WS(T, 1) WS(T, 2) WS(T, 4) - } else { - WS(T, 2) WS(T, 4) WS(T, 8) - } - std::fprintf(stderr, "blochsim sweep: no variant %d,%d,%d\n", y, w, occ); - return 1; -} - -} // namespace - -int real_forward(const layout_real::Params& p, cudaStream_t stream) { - return dispatch("BLOCHSIM_REAL_SWEEP", 8, false, p, stream); -} - -int real_jvp(const layout_real::Params& p, cudaStream_t stream) { - return dispatch("BLOCHSIM_REAL_JVP_SWEEP", 4, true, p, stream); -} - -} // namespace blochsim_layout diff --git a/benchmarks/sweep_real.py b/benchmarks/sweep_real.py deleted file mode 100644 index 660bfecb..00000000 --- a/benchmarks/sweep_real.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Time MRF-FISP (500 TRs, 32 states) forward and T1/T2 Jacobian per kernel variant. - -Run from the blochsim-bench env with a scratch build of the sweep branch: - python sweep.py --atoms 100000 [--profile] -""" - -import argparse -import itertools -import os -import sys - -import torch - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - - -def main(): - cli = argparse.ArgumentParser() - cli.add_argument("--atoms", type=int, default=100_000) - cli.add_argument("--length", type=int, default=500) - cli.add_argument("--states", type=int, default=32) - cli.add_argument("--repeats", type=int, default=30) - cli.add_argument("--profile", action="store_true") - cli.add_argument("--no-sweep", action="store_true") - args = cli.parse_args() - - from blochsim.simulators import MRFSimulator - - device = torch.device("cuda") - from _common import flip_train, tissue_grid - - flip = torch.tensor(flip_train(args.length), dtype=torch.float32, device=device) - t1, t2 = tissue_grid(args.atoms) - T1 = torch.tensor(t1, dtype=torch.float32, device=device) - T2 = torch.tensor(t2, dtype=torch.float32, device=device) - seq = MRFSimulator(flip=flip, TR=10.0, states=args.states).to(device) - seq.simulate(T1=T1[:1], T2=T2[:1]) - seq.jacobian(("T1", "T2"), T1=T1[:1], T2=T2[:1]) - - def timed(fn, repeats=args.repeats): - for _ in range(3): - fn() - torch.cuda.synchronize() - times = [] - for _ in range(repeats): - a = torch.cuda.Event(enable_timing=True) - b = torch.cuda.Event(enable_timing=True) - a.record() - fn() - b.record() - torch.cuda.synchronize() - times.append(a.elapsed_time(b)) - times.sort() - return times[len(times) // 2] - - fwd = lambda: seq.simulate(T1=T1, T2=T2) # noqa: E731 - jac = lambda: seq.jacobian(("T1", "T2"), T1=T1, T2=T2) # noqa: E731 - reference = fwd() - print(f"forward default: {timed(fwd):.2f} ms", flush=True) - print(f"jacobian default: {timed(jac, 10):.2f} ms", flush=True) - print(f"checksum {complex(reference.sum().cpu())}", flush=True) - - if args.profile: - from torch.profiler import ProfilerActivity, profile - - for name, fn in (("forward", fwd), ("jacobian", jac)): - with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof: - for _ in range(5): - fn() - torch.cuda.synchronize() - print(f"== {name} (5 calls)") - print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=25)) - - if args.no_sweep: - return - for y, w, occ in itertools.product((2, 4, 8), (2, 4, 8), (0, 16, 24, 32)): - os.environ["BLOCHSIM_REAL_SWEEP"] = f"{y},{w},{occ}" - try: - out = fwd() - err = (out - reference).abs().max().item() - print(f"fwd Y={y} W={w} occ={occ}: {timed(fwd):.2f} ms maxerr {err:.1e}", flush=True) - except Exception as e: # noqa: BLE001 - print(f"fwd Y={y} W={w} occ={occ}: failed {e}", flush=True) - os.environ.pop("BLOCHSIM_REAL_SWEEP") - for y, w, occ in itertools.product((1, 2, 4), (2, 4, 8), (0, 16, 24, 32)): - os.environ["BLOCHSIM_REAL_JVP_SWEEP"] = f"{y},{w},{occ}" - try: - print(f"jvp Y={y} W={w} occ={occ}: jacobian {timed(jac, 10):.2f} ms", flush=True) - except Exception as e: # noqa: BLE001 - print(f"jvp Y={y} W={w} occ={occ}: failed {e}", flush=True) - - -if __name__ == "__main__": - main() From ecafc18cd93a87cd0f61d23350ea8d579a064b7e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 17:08:18 +0000 Subject: [PATCH 8/8] test: a fingerprinting sample after an inversion is real and turned by the flip Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFB6QP4KCaPBFdw9c6SoRd --- tests/simulators/test_epg_trains.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/simulators/test_epg_trains.py b/tests/simulators/test_epg_trains.py index eafb1831..bb01380c 100644 --- a/tests/simulators/test_epg_trains.py +++ b/tests/simulators/test_epg_trains.py @@ -283,3 +283,30 @@ def test_trains_with_their_own_tr_each_settle_as_if_played_alone(): flip=flips_deg[train], ESP=10.0, TR=float(tr_ms[train, 0]) ).simulate(**TISSUE) _close(np.asarray(batched)[train], alone) + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("flip", [10.0, 60.0]) +def test_a_fingerprinting_sample_after_an_inversion_is_real_and_turned_by_the_flip( + device, flip +): + import torch + + from blochsim.simulators import MRFSimulator + + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("no CUDA device") + tissue = torch.ones(64, device=device) + flips = torch.tensor([flip, flip]) + signal = MRFSimulator(flip=flips, TR=1e-6).simulate( + T1=1e12 * tissue, T2=1e12 * tissue + ) + first = math.sin(math.radians(flip)) + # The first pulse turns the inverted magnetization into the transverse + # plane; the second reads what the first left along z, and the spoiler has + # moved the first pulse's transverse state off the echo. + expected = torch.tensor( + [first, first * math.cos(math.radians(flip))], device=device + ).expand(signal.shape) + torch.testing.assert_close(signal.abs(), expected) + assert torch.all(signal.imag.abs() < 1e-6)