diff --git a/src/blochsim/_epg_kernels.hpp b/src/blochsim/_epg_kernels.hpp index e3f434d9..be28c4b1 100644 --- a/src/blochsim/_epg_kernels.hpp +++ b/src/blochsim/_epg_kernels.hpp @@ -9154,7 +9154,7 @@ BSK_HD void _epg_real_vjp_jvp_kernel(float* t1, float* t2, float* m0, float* b1, bsk::atomic_add(((grad_tissue_tangent + ((7 + past_transmit) * atom_count)) + atom), grad_damping_tangent, active_atom); } -BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { +BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { bsk::V alpha{}; bsk::V atom_b1{}; bsk::V atom_damping{}; @@ -9302,8 +9302,13 @@ BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* auto out_ = bsk::ld((output_index + event)); auto output_offset = ((problem * output_count) + out_); auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0)); - bsk::st(((output_real + output_offset) + state), empty, output_mask); - bsk::st(((output_imag + output_offset) + state), (atom_m0 * plus), output_mask); + auto output_at = (output + ((output_offset + state) * 2)); + // The signal is i times the state, before its factor. + auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2))); + auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1)); + auto held = (atom_m0 * plus); + bsk::st(output_at, (-factor_imag * held), output_mask); + bsk::st((output_at + 1), (factor_real * held), output_mask); } if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) { auto t2_ = _shift_real(plus, minus, state, state_mask, state_count); @@ -12008,7 +12013,7 @@ BSK_HD void _epg_vjp_kernel(float* t1, float* t2, float* m0, float* b1, float* b }); } -BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { +BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) { bsk::V atom_b1{}; bsk::V atom_damping{}; bsk::V atom_inversion{}; @@ -12201,8 +12206,13 @@ BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, flo auto output_offset = ((problem * output_count) + out_); auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0)); auto signal_imag = ((dot_m0 * plus) + (atom_m0 * dot_plus)); - bsk::st(((output_real + output_offset) + state), empty, output_mask); - bsk::st(((output_imag + output_offset) + state), signal_imag, output_mask); + auto output_at = (output + ((output_offset + state) * 2)); + // The signal is i times the state, before its factor. + auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2))); + auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1)); + auto held = signal_imag; + bsk::st(output_at, (-factor_imag * held), output_mask); + bsk::st((output_at + 1), (factor_real * held), output_mask); } if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) { auto t4_ = _shift_real(plus, minus, state, state_mask, state_count); diff --git a/src/blochsim/_kernels.hpp b/src/blochsim/_kernels.hpp index 74e79831..3e7df8d9 100644 --- a/src/blochsim/_kernels.hpp +++ b/src/blochsim/_kernels.hpp @@ -818,8 +818,8 @@ inline constexpr KernelInfo KERNELS[] = { {"_epg_vjp_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,grad_pair_value,grad_pair_tangent,dot_t1,dot_t2,dot_m0,dot_b1,dot_b1_phase,dot_b0,dot_inversion_efficiency,dot_diffusion,dot_velocity,dot_bound_fraction,dot_exchange_rate,dot_t1_bound,dot_pool_b_fraction,dot_pool_b_exchange,dot_t1_pool_b,dot_t2_pool_b,dot_pool_b_shift,dot_duration,dot_flip,dot_phase,duration_row,pool_table,pool_bars,pool_durations,row_count,grad_output_real,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_phase_value,grad_phase_tangent,grad_duration_value,grad_duration_tangent,trajectory_vr,trajectory_vi,trajectory_tr,trajectory_ti,problem_base,problem_end,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,directed,off_axis,moving,diffusing,transmit,density,inverting,broadened,lineshape_bins,pools,narrow,tabulated,recording,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppippppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiiii", 105, 106, -1, BLOCHSIM_LANES__epg_vjp_jvp_kernel}, {"_epg_real_vjp_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,dot_t1,dot_t2,dot_m0,dot_b1,dot_inversion_efficiency,dot_diffusion,dot_duration,dot_flip,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_duration_value,grad_duration_tangent,trajectory_value,trajectory_tangent,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppiiiiiiiiiiiiiiiii", 44, 45, -1, BLOCHSIM_LANES__epg_real_vjp_jvp_kernel}, {"_epg_real_vjp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,grad_output_imag,grad_tissue,grad_flip,grad_duration,trajectory_value,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppiiiiiiiiiiiiiiiii", 32, 33, -1, BLOCHSIM_LANES__epg_real_vjp_kernel}, - {"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel}, - {"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel}, + {"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel}, + {"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel}, {"_epg_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,bound_exchange,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,phase_cos,phase_sin,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 65, 66, -1, BLOCHSIM_LANES__epg_kernel}, {"_epg_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_b1_phase,tangent_b0,tangent_inversion_efficiency,tangent_diffusion,tangent_velocity,tangent_bound_fraction,tangent_exchange_rate,tangent_t1_bound,tangent_pool_b_fraction,tangent_pool_b_exchange,tangent_t1_pool_b,tangent_t2_pool_b,tangent_pool_b_shift,tangent_duration,tangent_flip,tangent_phase,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 84, 85, -1, BLOCHSIM_LANES__epg_jvp_kernel}, {"_pooled_kernel", "m0,b1,b1_phase,b0,efficiency,diffusion,velocity,dm0,db1,db1_phase,db0,defficiency,ddiffusion,dvelocity,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,dduration,dflip,dphase,table,dtable,pool_index,profile,profile_index,lineshape,pairs,pair_index,dpairs,output_real,output_imag,trajectory,base,atom_count,event_count,output_count,state_count,rows,flow_scale,washout_scale,profile_step,lineshape_step,locations,profile_bins,lineshape_bins,n,m,blocks,planes,atom_stride,shimmed,profiled,dynamic,directed_pairs,directed_table,following,keep,off_axis,moving,diffusing,transmit,density,inverting,P,S", "ppppppppppppppppppppppppppppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiii", 70, 69, 69, BLOCHSIM_LANES__pooled_kernel}, diff --git a/src/blochsim/_layout.cu b/src/blochsim/_layout.cu index 9f918e4a..6c541df5 100644 --- a/src/blochsim/_layout.cu +++ b/src/blochsim/_layout.cu @@ -140,8 +140,8 @@ layout_real::Params real_params(const Reader& r) { p.output_index = r.ints("output_index"); p.shim_index = r.ints("shim_index"); p.action = static_cast(r.a[r.at("action")].p); - p.output_real = r.outputs("output_real"); - p.output_imag = r.outputs("output_imag"); + p.output = r.outputs("output"); + p.sample_factor = r.floats("sample_factor"); p.atom_count = static_cast(r.integer("atom_count")); p.train_count = static_cast(r.integer("train_count")); p.event_count = static_cast(r.integer("event_count")); diff --git a/src/blochsim/_layout_real.cu b/src/blochsim/_layout_real.cu index ad48a785..cbe83bf7 100644 --- a/src/blochsim/_layout_real.cu +++ b/src/blochsim/_layout_real.cu @@ -4,8 +4,10 @@ namespace blochsim_layout { namespace { +// Held to the registers that let eight programs share an SM; uncapped, nvcc +// spends more and fewer programs hide each other's latency. template -__global__ void __launch_bounds__(32 * WARPS) real_kernel(layout_real::Params p) { +__global__ void __launch_bounds__(32 * WARPS, 8) real_kernel(layout_real::Params p) { layout_real::real_loop(p); } diff --git a/src/blochsim/_layout_real.hpp b/src/blochsim/_layout_real.hpp index 73fbb16f..b2e68798 100644 --- a/src/blochsim/_layout_real.hpp +++ b/src/blochsim/_layout_real.hpp @@ -23,7 +23,10 @@ struct Params { const float *d_t1, *d_t2, *d_m0, *d_b1, *d_inversion_efficiency, *d_diffusion, *d_duration, *d_flip; const int *kind, *output_index, *shim_index; const unsigned char* action; - float *output_real, *output_imag; + // Each sample as a complex number, real then imaginary; and the complex + // factor each sample is multiplied by, or null for none. + float* output; + const float* sample_factor; int atom_count, train_count, event_count, output_count, state_count, width; bool single_train, atom_stride, shimmed, diffusing, transmit, density, inverting; }; @@ -123,12 +126,15 @@ __device__ __forceinline__ void real_loop(const Params& p) { float held[Y]; int run_start = 0, run_len = 0; auto flush = [&]() { + // The signal is i times what a thread holds, before its factor. + const float2 factor = p.sample_factor && state < run_len + ? reinterpret_cast(p.sample_factor)[run_start + state] + : make_float2(1.0f, 0.0f); #pragma unroll for (int y = 0; y < Y; ++y) { if (state < run_len && active[y]) { const long at_out = static_cast(problem[y]) * p.output_count + run_start + state; - p.output_real[at_out] = 0.0f; - p.output_imag[at_out] = held[y]; + reinterpret_cast(p.output)[at_out] = make_float2(-factor.y * held[y], factor.x * held[y]); } } run_len = 0; diff --git a/src/blochsim/_model/_signal.py b/src/blochsim/_model/_signal.py index 334fb55b..f51fb413 100644 --- a/src/blochsim/_model/_signal.py +++ b/src/blochsim/_model/_signal.py @@ -229,16 +229,23 @@ def along(*inputs: torch.Tensor) -> torch.Tensor: return self._shaped(self.evaluate(declared, **sequence), batch) forward_mode() - columns = [] - signal = None - for name in names: - tangents = tuple( - torch.ones_like(value) if other == name else torch.zeros_like(value) - for other, value in zip(names, primals, strict=True) + # One direction per property, batched: the signal is simulated once and + # only the directional derivatives are taken per property. + directions = tuple( + torch.stack( + [ + torch.ones_like(value) if other == name else torch.zeros_like(value) + for name in names + ] ) - signal, column = torch.func.jvp(along, primals, tangents) - columns.append(column) - jacobian = columns[0] if isinstance(diff, str) else torch.stack(columns, dim=-2) + for other, value in zip(names, primals, strict=True) + ) + + def column(*tangents: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return torch.func.jvp(along, primals, tangents) + + signal, columns = torch.func.vmap(column, out_dims=(None, 0))(*directions) + jacobian = columns[0] if isinstance(diff, str) else columns.movedim(0, -2) return like(signal, backend), like(jacobian, backend) # -- what a subclass does not have to write ----------------------------- diff --git a/src/blochsim/_model/_state_machine.py b/src/blochsim/_model/_state_machine.py index a3090e81..466c889d 100644 --- a/src/blochsim/_model/_state_machine.py +++ b/src/blochsim/_model/_state_machine.py @@ -529,6 +529,9 @@ class Simulator(_SignalModel): across_slice: Any = None crusher_dephasing_rad: float = 0.0 voxel_size_m: float | None = None + # The constant every recorded sample is multiplied by: the receiver phase + # that puts a sequence's signal on the axis its readers expect. + receiver: complex = 1.0 # Every attribute the constructor sets has a value here as well, so a # closed form that writes an __init__ of its own -- taking the two blocks @@ -1006,6 +1009,7 @@ def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tens nstates=states, events=events, across_slice=profile, + _receiver=self.receiver, **settings, ) .signal diff --git a/src/blochsim/sequence/_accelerators.py b/src/blochsim/sequence/_accelerators.py index 17f9933f..a5bf6fb7 100644 --- a/src/blochsim/sequence/_accelerators.py +++ b/src/blochsim/sequence/_accelerators.py @@ -601,14 +601,28 @@ def _one_axis(packed: _PackedEvents) -> tuple[_PackedEvents, torch.Tensor | None if corrections is None: return packed, None out_by, flips, samples = corrections - return ( - replace( - packed, - flip=(packed.flip * flips).contiguous(), - phase=(packed.phase - out_by * torch.pi).contiguous(), - ), - samples, + + def turned() -> tuple[torch.Tensor, torch.Tensor]: + return ( + (packed.flip * flips).contiguous(), + (packed.phase - out_by * torch.pi).contiguous(), + ) + + # The turned buffers are kept with the stream they came from, so what is + # remembered against them downstream -- the pulse axis above all, which is + # a device round trip -- is read once per stream rather than once per call. + # A buffer carrying a graph is turned afresh, so no call backpropagates + # through another's. + flip, phase = ( + turned() + if packed.flip.requires_grad or packed.phase.requires_grad + else _remembered( + "turned onto one axis", + (packed.kind, packed.flip, packed.phase, packed.action), + turned, + ) ) + return replace(packed, flip=flip, phase=phase), samples def simulate_native( @@ -630,6 +644,7 @@ def simulate_native( record_every: bool = False, pools: int = 0, extra_pools: tuple[torch.Tensor, ...] | None = None, + receiver: complex = 1.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] | None: """Run a fused CPU/CUDA state machine with explicit AD rules. @@ -651,6 +666,8 @@ def simulate_native( what is skipped is walking the events again to rebuild what only their values changed. + ``receiver`` is the constant every recorded sample is multiplied by. + ``pools`` counts the chemically exchanging pools the tissue carries from pool B on, and ``extra_pools`` holds the prepared buffers of pools C to E. From :data:`~blochsim.sequence._pools.TABULATE_FROM` pools on, the kernels @@ -766,6 +783,7 @@ def simulate_native( features, pool_values, pool_index, + _sample_factor(samples, receiver, packed.output_count, device), ) leading = (packed.train_count,) if packed.is_batched else () if locations > 1: @@ -775,8 +793,6 @@ def simulate_native( signal = signal.reshape(*leading, *output_shape, packed.output_count) if dephasing is not None: signal = signal * dephasing - if samples is not None: - signal = signal * samples return ( signal, packed.time_us, @@ -1491,6 +1507,7 @@ def forward( features: frozenset[str] | None, pool_values: torch.Tensor | None = None, pool_index: torch.Tensor | None = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: tissue = ( t1, @@ -1535,6 +1552,7 @@ def forward( exchanging=exchanging, features=features, pools=pools_of(pool_values, pool_index, exchanging), + factor=factor, ) @staticmethod @@ -1559,6 +1577,9 @@ def setup_context(ctx: Any, inputs: tuple[Any, ...], _output: torch.Tensor) -> N ctx.pool_index = ( inputs[_PACKED_COUNT + 11] if len(inputs) > _PACKED_COUNT + 11 else None ) + ctx.factor = ( + inputs[_PACKED_COUNT + 12] if len(inputs) > _PACKED_COUNT + 12 else None + ) @staticmethod @_under_recorded_policy @@ -1570,7 +1591,7 @@ def jvp(ctx: Any, *tangents: torch.Tensor | None) -> torch.Tensor: else tangents[index].to(saved[index]).contiguous() for index in _FLOAT_INPUTS ) - return _NativeEpgJvp.apply( + tangent = _NativeEpgJvp.apply( *saved, *float_tangents, ctx.state_count, @@ -1593,12 +1614,15 @@ def jvp(ctx: Any, *tangents: torch.Tensor | None) -> torch.Tensor: ctx.pool_values, ), ) + return tangent if ctx.factor is None else tangent * ctx.factor @staticmethod @_under_recorded_policy def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[Any, ...]: saved = ctx.saved_tensors wanted = _wanted(ctx.needs_input_grad) + if ctx.factor is not None: + grad_output = grad_output * ctx.factor.conj() fused = _NativeEpgVjp.apply( *saved, grad_output, @@ -1637,6 +1661,7 @@ def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[Any, ...]: None, pool_grad if _needs(ctx, _PACKED_COUNT + 10) else None, None, + None, ) @staticmethod @@ -3047,6 +3072,7 @@ def _run_packed( dynamic: Any = None, features: frozenset[str] | None = None, pools: PoolTables | None = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: """Run the forward state machine. @@ -3060,6 +3086,9 @@ def _run_packed( ``features`` names the terms the tissue gives anything to do, and the kernels carry those and no others. ``None`` leaves every term in. + + ``factor`` is a complex number per output sample that multiplies the + signal. """ profile = _tables(profile, events, dynamic) if real_axis is None: @@ -3108,6 +3137,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=piece["dynamic"], features=features, pools=piece["pools"], + factor=_on(factor, device), ), ) from ._epg_gpu import simulate @@ -3126,6 +3156,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=piece["dynamic"], features=features, pools=piece["pools"], + factor=_on(factor, device), ), ) @@ -3163,6 +3194,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: lineshape=lineshape, exchanging=exchanging, features=features, + factor=_on(factor, device), ), ) for begin, end, device in shards @@ -3181,6 +3213,7 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: dynamic=dynamic, features=features, pools=pools, + factor=factor, ) from blochsim import _epg_cpu @@ -3225,7 +3258,33 @@ def run(device: torch.device, **piece: Any) -> tuple[torch.Tensor]: feature_mask(features, geometry), *_pool_counts(pools), ) - return torch.complex(output_real, output_imag) + signal = torch.complex(output_real, output_imag) + return signal if factor is None else signal * factor + + +def _sample_factor( + samples: torch.Tensor | None, + receiver: complex, + output_count: int, + device: torch.device, +) -> torch.Tensor | None: + """The complex number each recorded sample is multiplied by, or ``None``. + + ``samples`` is the sign the packing moved off the pulses, one per sample, + and ``receiver`` the constant the caller asked for. + """ + if samples is None and receiver == 1: + return None + if samples is None: + return torch.full( + (output_count,), complex(receiver), dtype=torch.complex64, device=device + ) + return (samples * complex(receiver)).to(torch.complex64).contiguous() + + +def _on(value: torch.Tensor | None, device: torch.device) -> torch.Tensor | None: + """``value`` on ``device``, or ``None``.""" + return None if value is None else value.to(device) def _card_vjp( diff --git a/src/blochsim/sequence/_epg_gpu.py b/src/blochsim/sequence/_epg_gpu.py index eb84c3a1..49eca722 100644 --- a/src/blochsim/sequence/_epg_gpu.py +++ b/src/blochsim/sequence/_epg_gpu.py @@ -364,16 +364,20 @@ def simulate( dynamic: Any = None, features: frozenset[str] | None = None, pools: Any = None, + factor: torch.Tensor | None = None, ) -> torch.Tensor: """Run a packed state machine on CUDA and return complex signals. ``real_axis`` of 1 selects the real-subspace kernel; see ``real_subspace_axis`` for when that is legitimate. + + ``factor`` is a complex number per output sample, ``(output_count,)``, + that multiplies the signal; ``None`` leaves it as the kernels produce it. """ if pools is not None: from . import _pools_gpu - return _pools_gpu.simulate( + signal = _pools_gpu.simulate( tissue, events, state_count=state_count, @@ -385,13 +389,25 @@ def simulate( features=features, pools=pools, ) + return signal if factor is None else signal * factor train_count = _train_count(events) atom_count = tissue[0].numel() - output_real = torch.empty( - _output_shape(train_count, atom_count, output_count), - dtype=torch.float32, - device=tissue[0].device, - ) + shape = _output_shape(train_count, atom_count, output_count) + if real_axis == 1: + signal = torch.empty(shape, dtype=torch.complex64, device=tissue[0].device) + _simulate_real( + tissue, + events, + signal, + state_count=state_count, + output_count=output_count, + atom_count=atom_count, + geometry=geometry, + features=features, + factor=factor, + ) + return signal + output_real = torch.empty(shape, dtype=torch.float32, device=tissue[0].device) output_imag = torch.empty_like(output_real) simulate_into( tissue, @@ -400,7 +416,6 @@ def simulate( output_imag, state_count=state_count, output_count=output_count, - real_axis=real_axis, atom_count=atom_count, geometry=geometry, profile=profile, @@ -409,7 +424,60 @@ def simulate( dynamic=dynamic, features=features, ) - return torch.complex(output_real, output_imag) + signal = torch.complex(output_real, output_imag) + return signal if factor is None else signal * factor + + +def _simulate_real( + tissue: tuple[torch.Tensor, ...], + events: tuple[torch.Tensor, ...], + signal: torch.Tensor, + *, + state_count: int, + output_count: int, + atom_count: int, + geometry: Geometry, + features: frozenset[str] | None, + factor: torch.Tensor | None, +) -> None: + """Run the real-subspace kernel into a complex signal the caller owns. + + The kernel writes each sample as a complex number with its ``factor`` + already applied. + """ + t1, t2, m0, b1, _, _, inversion_efficiency, diffusion = tissue[:8] + duration, kind, flip, _, action, output_index, shim_index = events[:7] + train_count = _train_count(events) + block_states = next_power_of_2(state_count) + problems = _problems_per_program(block_states, _epg_real_kernel) + grid = (cdiv(train_count * atom_count, problems),) + _epg_real_kernel[grid]( + t1, + t2, + m0, + b1, + inversion_efficiency, + diffusion, + duration, + kind, + flip, + action, + output_index, + shim_index, + torch.view_as_real(signal), + None if factor is None else torch.view_as_real(factor.to(torch.complex64)), + atom_count, + train_count, + kind.numel(), + output_count, + state_count=state_count, + single_train=train_count == 1, + atom_stride=_atom_stride(tissue), + shimmed=_shim_count(tissue) > 1, + **_only_scalars(_feature_flags(features, geometry)), + block_states=block_states, + problems=problems, + ) def simulate_into( @@ -420,7 +488,6 @@ def simulate_into( *, state_count: int, output_count: int, - real_axis: int | None, atom_count: int, geometry: Geometry = NO_GEOMETRY, profile: Any = None, @@ -473,9 +540,7 @@ def simulate_into( pools = _pool_flag(lineshape, exchanging) block_states = next_power_of_2(state_count) total = train_count * atom_count - problems = _problems_per_program( - block_states, _epg_real_kernel if real_axis == 1 else _epg_kernel - ) + problems = _problems_per_program(block_states, _epg_kernel) grid = (cdiv(total, problems),) # A kernel argument has to be a tensor even where the branch reading it is # compiled out, so an unprofiled launch passes one it already has. @@ -498,36 +563,6 @@ def simulate_into( phase_cos = torch.cos(phase.double()).to(torch.float32) phase_sin = torch.sin(phase.double()).to(torch.float32) - if real_axis == 1: - _epg_real_kernel[grid]( - t1, - t2, - m0, - b1, - inversion_efficiency, - diffusion, - duration, - kind, - flip, - action, - output_index, - shim_index, - output_real, - output_imag, - atom_count, - train_count, - kind.numel(), - output_count, - state_count=state_count, - single_train=train_count == 1, - atom_stride=_atom_stride(tissue), - shimmed=_shim_count(tissue) > 1, - **_only_scalars(_feature_flags(features, geometry)), - block_states=block_states, - problems=problems, - ) - return - _epg_kernel[grid]( t1, t2, @@ -638,11 +673,23 @@ def simulate_jvp( ) train_count = _train_count(events) atom_count = tissue[0].numel() - output_real = torch.empty( - _output_shape(train_count, atom_count, output_count), - dtype=torch.float32, - device=tissue[0].device, - ) + shape = _output_shape(train_count, atom_count, output_count) + if real_axis == 1: + tangent = torch.empty(shape, dtype=torch.complex64, device=tissue[0].device) + _simulate_real_jvp( + tissue, + events, + tissue_tangents, + event_tangents, + tangent, + state_count=state_count, + output_count=output_count, + atom_count=atom_count, + geometry=geometry, + features=features, + ) + return tangent + output_real = torch.empty(shape, dtype=torch.float32, device=tissue[0].device) output_imag = torch.empty_like(output_real) simulate_jvp_into( tissue, @@ -653,7 +700,6 @@ def simulate_jvp( output_imag, state_count=state_count, output_count=output_count, - real_axis=real_axis, atom_count=atom_count, geometry=geometry, profile=profile, @@ -666,6 +712,64 @@ def simulate_jvp( return torch.complex(output_real, output_imag) +def _simulate_real_jvp( + tissue: tuple[torch.Tensor, ...], + events: tuple[torch.Tensor, ...], + tissue_tangents: tuple[torch.Tensor, ...], + event_tangents: tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tangent: torch.Tensor, + *, + state_count: int, + output_count: int, + atom_count: int, + geometry: Geometry, + features: frozenset[str] | None, +) -> None: + """Run the real-subspace Jacobian-vector product into a complex tangent.""" + t1, t2, m0, b1, _, _, inversion_efficiency, diffusion = tissue[:8] + duration, kind, flip, _, action, output_index, shim_index = events[:7] + tangent_duration, tangent_flip, _ = event_tangents + train_count = _train_count(events) + block_states = next_power_of_2(state_count) + problems = _problems_per_program(block_states, _epg_real_jvp_kernel) + grid = (cdiv(train_count * atom_count, problems),) + _epg_real_jvp_kernel[grid]( + t1, + t2, + m0, + b1, + inversion_efficiency, + diffusion, + duration, + kind, + flip, + action, + output_index, + shim_index, + tissue_tangents[0], + tissue_tangents[1], + tissue_tangents[2], + tissue_tangents[3], + tissue_tangents[6], + tissue_tangents[7], + tangent_duration, + tangent_flip, + torch.view_as_real(tangent), + None, + atom_count, + train_count, + kind.numel(), + output_count, + state_count=state_count, + single_train=train_count == 1, + atom_stride=_atom_stride(tissue, tissue_tangents), + shimmed=_shim_count(tissue) > 1, + **_only_scalars(_feature_flags(features, geometry)), + block_states=block_states, + problems=problems, + ) + + def simulate_jvp_into( tissue: tuple[torch.Tensor, ...], events: tuple[torch.Tensor, ...], @@ -676,7 +780,6 @@ def simulate_jvp_into( *, state_count: int, output_count: int, - real_axis: int | None, atom_count: int, geometry: Geometry = NO_GEOMETRY, profile: Any = None, @@ -726,49 +829,9 @@ def simulate_jvp_into( shims = _shim_count(tissue) block_states = next_power_of_2(state_count) total = train_count * atom_count - problems = _problems_per_program( - block_states, _epg_real_jvp_kernel if real_axis == 1 else _epg_jvp_kernel - ) + problems = _problems_per_program(block_states, _epg_jvp_kernel) grid = (cdiv(total, problems),) - if real_axis == 1: - _epg_real_jvp_kernel[grid]( - t1, - t2, - m0, - b1, - inversion_efficiency, - diffusion, - duration, - kind, - flip, - action, - output_index, - shim_index, - tissue_tangents[0], - tissue_tangents[1], - tissue_tangents[2], - tissue_tangents[3], - tissue_tangents[6], - tissue_tangents[7], - tangent_duration, - tangent_flip, - output_real, - output_imag, - atom_count, - train_count, - kind.numel(), - output_count, - state_count=state_count, - single_train=train_count == 1, - atom_stride=_atom_stride(tissue, tissue_tangents), - shimmed=shims > 1, - **_only_scalars(_feature_flags(features, geometry)), - block_states=block_states, - problems=problems, - ) - return - table = None if profile is None else profile.packed(t1.device) pairs = None if dynamic is None else dynamic.packed(t1.device) pair_rows = ( diff --git a/src/blochsim/sequence/_simulation.py b/src/blochsim/sequence/_simulation.py index 6fb96bce..a3b8e0fc 100644 --- a/src/blochsim/sequence/_simulation.py +++ b/src/blochsim/sequence/_simulation.py @@ -374,6 +374,7 @@ def simulate( device: torch.device | str | None = None, events: Any = None, _every: bool = False, + _receiver: complex = 1.0, ) -> SimulationResult: """Walk an event stream and record selected ADC signals. @@ -400,7 +401,7 @@ def simulate( if rf_raster_time_s is None: rf_raster_time_s = description.rf_raster_time_s if repetitions == AUTO: - return self._settled( + settled = self._settled( description, tissue, record=record, @@ -409,6 +410,9 @@ def simulate( rf_raster_time_s=rf_raster_time_s, device=device, ) + if _receiver == 1: + return settled + return replace(settled, signal=settled.signal * _receiver) repetitions = _as_integer(repetitions, "repetitions") if repetitions < 1: raise ValueError("repetitions must be positive") @@ -487,6 +491,7 @@ def simulate( record_every=_every, pools=pools, extra_pools=extra_pools, + receiver=_receiver, ) if accelerated is None: raise RuntimeError( @@ -793,6 +798,11 @@ def _prepare_pools( def _as_float_tensor(value: Any, device: torch.device) -> torch.Tensor: if isinstance(value, torch.Tensor): return value.to(device=device, dtype=torch.float32) + if isinstance(value, (int, float)): + # Filled where it lives: a tissue carries some thirty properties left + # at a plain number, and a copy of each from the host would wait on + # the device every call. + return torch.full((), float(value), dtype=torch.float32, device=device) return torch.as_tensor(value, dtype=torch.float32, device=device) diff --git a/src/blochsim/simulators/mpnrage.py b/src/blochsim/simulators/mpnrage.py index 6eef7f74..157cbeac 100644 --- a/src/blochsim/simulators/mpnrage.py +++ b/src/blochsim/simulators/mpnrage.py @@ -4,8 +4,6 @@ __all__ = ["MPnRAGESimulator"] -from collections.abc import Mapping -from typing import Any import numpy.typing as npt import torch @@ -71,6 +69,7 @@ class MPnRAGESimulator(Simulator): operators=SPOILED, ) states = 1 + receiver = 1j def layout( self, @@ -107,7 +106,3 @@ def layout( self.operators.readout(turns[index], duration_s=spacing_s[index]) ) return parts - - def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tensor: - """Return the signal at every shot.""" - return 1j * super().evaluate(properties, **sequence) diff --git a/src/blochsim/simulators/mprage.py b/src/blochsim/simulators/mprage.py index 2b3d522d..3ad18de8 100644 --- a/src/blochsim/simulators/mprage.py +++ b/src/blochsim/simulators/mprage.py @@ -74,6 +74,7 @@ class MPRAGESimulator(Simulator): ) states = 1 record = "acquired" + receiver = 1j def layout( self, @@ -125,7 +126,7 @@ def layout( def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tensor: """Return the acquired sample.""" - return 1j * super().evaluate(properties, **sequence)[..., 0] + return super().evaluate(properties, **sequence)[..., 0] # %% private module subroutines diff --git a/src/blochsim/simulators/mrf.py b/src/blochsim/simulators/mrf.py index 95433099..6ede62f1 100644 --- a/src/blochsim/simulators/mrf.py +++ b/src/blochsim/simulators/mrf.py @@ -74,6 +74,7 @@ class MRFSimulator(Simulator): operators=UNBALANCED, ) states = 10 + receiver = 1j def layout( self, @@ -101,4 +102,4 @@ def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tens signal = super().evaluate(properties, **sequence) played = self.played(**sequence) echoes = torch.atleast_1d(played["flip"]).shape[-1] - return 1j * signal[..., -echoes:] + return signal[..., -echoes:] diff --git a/tests/sequence/test_parameters.py b/tests/sequence/test_parameters.py index 6a7b1b70..4edd70c4 100644 --- a/tests/sequence/test_parameters.py +++ b/tests/sequence/test_parameters.py @@ -202,10 +202,11 @@ def record(needs_input_grad): finally: _accelerators._wanted = original - # The twelve trailing arguments -- state count, output count, thread count, - # the sequence geometry, the transition table, the per-voxel rotations and - # the row each event reads of them, the semisolid pool's lineshape, how the - # chemically exchanging pools are carried, which terms the tissue asks for, - # and the pool tables with the row each event reads of them -- follow the - # packed buffers. - assert widths and all(width == PACKED_COUNT + 12 for width in widths) + # The thirteen trailing arguments -- state count, output count, thread + # count, the sequence geometry, the transition table, the per-voxel + # rotations and the row each event reads of them, the semisolid pool's + # lineshape, how the chemically exchanging pools are carried, which terms + # the tissue asks for, the pool tables with the row each event reads of + # them, and the factor each sample is multiplied by -- follow the packed + # buffers. + assert widths and all(width == PACKED_COUNT + 13 for width in widths) diff --git a/tests/simulators/test_epg_trains.py b/tests/simulators/test_epg_trains.py index eafb1831..bb01380c 100644 --- a/tests/simulators/test_epg_trains.py +++ b/tests/simulators/test_epg_trains.py @@ -283,3 +283,30 @@ def test_trains_with_their_own_tr_each_settle_as_if_played_alone(): flip=flips_deg[train], ESP=10.0, TR=float(tr_ms[train, 0]) ).simulate(**TISSUE) _close(np.asarray(batched)[train], alone) + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("flip", [10.0, 60.0]) +def test_a_fingerprinting_sample_after_an_inversion_is_real_and_turned_by_the_flip( + device, flip +): + import torch + + from blochsim.simulators import MRFSimulator + + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("no CUDA device") + tissue = torch.ones(64, device=device) + flips = torch.tensor([flip, flip]) + signal = MRFSimulator(flip=flips, TR=1e-6).simulate( + T1=1e12 * tissue, T2=1e12 * tissue + ) + first = math.sin(math.radians(flip)) + # The first pulse turns the inverted magnetization into the transverse + # plane; the second reads what the first left along z, and the spoiler has + # moved the first pulse's transverse state off the echo. + expected = torch.tensor( + [first, first * math.cos(math.radians(flip))], device=device + ).expand(signal.shape) + torch.testing.assert_close(signal.abs(), expected) + assert torch.all(signal.imag.abs() < 1e-6)