Skip to content
Merged
22 changes: 16 additions & 6 deletions src/blochsim/_epg_kernels.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9154,7 +9154,7 @@ BSK_HD void _epg_real_vjp_jvp_kernel(float* t1, float* t2, float* m0, float* b1,
bsk::atomic_add(((grad_tissue_tangent + ((7 + past_transmit) * atom_count)) + atom), grad_damping_tangent, active_atom);
}

BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) {
BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) {
bsk::V<float, 2> alpha{};
bsk::V<float, 2> atom_b1{};
bsk::V<float, 2> atom_damping{};
Expand Down Expand Up @@ -9302,8 +9302,13 @@ BSK_HD void _epg_real_kernel(float* t1, float* t2, float* m0, float* b1, float*
auto out_ = bsk::ld((output_index + event));
auto output_offset = ((problem * output_count) + out_);
auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0));
bsk::st(((output_real + output_offset) + state), empty, output_mask);
bsk::st(((output_imag + output_offset) + state), (atom_m0 * plus), output_mask);
auto output_at = (output + ((output_offset + state) * 2));
// The signal is i times the state, before its factor.
auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2)));
auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1));
auto held = (atom_m0 * plus);
bsk::st(output_at, (-factor_imag * held), output_mask);
bsk::st((output_at + 1), (factor_real * held), output_mask);
}
if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) {
auto t2_ = _shift_real(plus, minus, state, state_mask, state_count);
Expand Down Expand Up @@ -12008,7 +12013,7 @@ BSK_HD void _epg_vjp_kernel(float* t1, float* t2, float* m0, float* b1, float* b
});
}

BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output_real, float* output_imag, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) {
BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, float* inversion_efficiency, float* diffusion, float* duration, std::int32_t* kind, float* flip, std::uint8_t* action, std::int32_t* output_index, std::int32_t* shim_index, float* tangent_t1, float* tangent_t2, float* tangent_m0, float* tangent_b1, float* tangent_inversion_efficiency, float* tangent_diffusion, float* tangent_duration, float* tangent_flip, float* output, float* sample_factor, bsk::index_t atom_count, bsk::index_t train_count, bsk::index_t event_count, bsk::index_t output_count, bsk::index_t state_count, bsk::index_t single_train, bsk::index_t atom_stride, bsk::index_t shimmed, bsk::index_t diffusing, bsk::index_t transmit, bsk::index_t density, bsk::index_t inverting, bsk::index_t block_states, bsk::index_t problems) {
bsk::V<float, 2> atom_b1{};
bsk::V<float, 2> atom_damping{};
bsk::V<float, 2> atom_inversion{};
Expand Down Expand Up @@ -12201,8 +12206,13 @@ BSK_HD void _epg_real_jvp_kernel(float* t1, float* t2, float* m0, float* b1, flo
auto output_offset = ((problem * output_count) + out_);
auto output_mask = bsk::band(bsk::band(active_atom, (state == 0)), (out_ >= 0));
auto signal_imag = ((dot_m0 * plus) + (atom_m0 * dot_plus));
bsk::st(((output_real + output_offset) + state), empty, output_mask);
bsk::st(((output_imag + output_offset) + state), signal_imag, output_mask);
auto output_at = (output + ((output_offset + state) * 2));
// The signal is i times the state, before its factor.
auto factor_real = (sample_factor == nullptr || out_ < 0) ? 1.0f : bsk::ld((sample_factor + (out_ * 2)));
auto factor_imag = (sample_factor == nullptr || out_ < 0) ? 0.0f : bsk::ld(((sample_factor + (out_ * 2)) + 1));
auto held = signal_imag;
bsk::st(output_at, (-factor_imag * held), output_mask);
bsk::st((output_at + 1), (factor_real * held), output_mask);
}
if (bsk::truth((bsk::truth((bsk::band(event_action, 2) != 0)) || bsk::truth((bsk::band(event_action, 16) != 0))))) {
auto t4_ = _shift_real(plus, minus, state, state_mask, state_count);
Expand Down
4 changes: 2 additions & 2 deletions src/blochsim/_kernels.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -818,8 +818,8 @@ inline constexpr KernelInfo KERNELS[] = {
{"_epg_vjp_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,grad_pair_value,grad_pair_tangent,dot_t1,dot_t2,dot_m0,dot_b1,dot_b1_phase,dot_b0,dot_inversion_efficiency,dot_diffusion,dot_velocity,dot_bound_fraction,dot_exchange_rate,dot_t1_bound,dot_pool_b_fraction,dot_pool_b_exchange,dot_t1_pool_b,dot_t2_pool_b,dot_pool_b_shift,dot_duration,dot_flip,dot_phase,duration_row,pool_table,pool_bars,pool_durations,row_count,grad_output_real,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_phase_value,grad_phase_tangent,grad_duration_value,grad_duration_tangent,trajectory_vr,trajectory_vi,trajectory_tr,trajectory_ti,problem_base,problem_end,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,directed,off_axis,moving,diffusing,transmit,density,inverting,broadened,lineshape_bins,pools,narrow,tabulated,recording,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppippppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiiii", 105, 106, -1, BLOCHSIM_LANES__epg_vjp_jvp_kernel},
{"_epg_real_vjp_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,dot_t1,dot_t2,dot_m0,dot_b1,dot_inversion_efficiency,dot_diffusion,dot_duration,dot_flip,grad_output_imag,grad_tissue_value,grad_tissue_tangent,grad_flip_value,grad_flip_tangent,grad_duration_value,grad_duration_tangent,trajectory_value,trajectory_tangent,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppiiiiiiiiiiiiiiiii", 44, 45, -1, BLOCHSIM_LANES__epg_real_vjp_jvp_kernel},
{"_epg_real_vjp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,grad_output_imag,grad_tissue,grad_flip,grad_duration,trajectory_value,problem_base,problem_end,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shim_rows,shimmed,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppiiiiiiiiiiiiiiiii", 32, 33, -1, BLOCHSIM_LANES__epg_real_vjp_kernel},
{"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel},
{"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output_real,output_imag,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel},
{"_epg_real_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppiiiiiiiiiiiiii", 26, 27, -1, BLOCHSIM_LANES__epg_real_kernel},
{"_epg_real_jvp_kernel", "t1,t2,m0,b1,inversion_efficiency,diffusion,duration,kind,flip,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_inversion_efficiency,tangent_diffusion,tangent_duration,tangent_flip,output,sample_factor,atom_count,train_count,event_count,output_count,state_count,single_train,atom_stride,shimmed,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppiiiiiiiiiiiiii", 34, 35, -1, BLOCHSIM_LANES__epg_real_jvp_kernel},
{"_epg_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,bound_exchange,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,phase_cos,phase_sin,action,output_index,shim_index,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "pppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 65, 66, -1, BLOCHSIM_LANES__epg_kernel},
{"_epg_jvp_kernel", "t1,t2,m0,b1,b1_phase,b0,inversion_efficiency,diffusion,velocity,bound_fraction,exchange_rate,t1_bound,pool_b_fraction,pool_b_exchange,t1_pool_b,t2_pool_b,pool_b_shift,duration,kind,flip,phase,action,output_index,shim_index,tangent_t1,tangent_t2,tangent_m0,tangent_b1,tangent_b1_phase,tangent_b0,tangent_inversion_efficiency,tangent_diffusion,tangent_velocity,tangent_bound_fraction,tangent_exchange_rate,tangent_t1_bound,tangent_pool_b_fraction,tangent_pool_b_exchange,tangent_t1_pool_b,tangent_t2_pool_b,tangent_pool_b_shift,tangent_duration,tangent_flip,tangent_phase,saturation,rf_frequency,profile,profile_index,lineshape,pairs,pair_index,pair_direction,duration_row,pool_table,output_real,output_imag,atom_count,train_count,event_count,output_count,flow_scale,washout_scale,profile_step,lineshape_step,state_count,single_train,atom_stride,shim_rows,shimmed,locations,profiled,profile_bins,dynamic,broadened,lineshape_bins,pools,narrow,tabulated,off_axis,moving,diffusing,transmit,density,inverting,block_states,problems", "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppiiiiffffiiiiiiiiiiiiiiiiiiiiii", 84, 85, -1, BLOCHSIM_LANES__epg_jvp_kernel},
{"_pooled_kernel", "m0,b1,b1_phase,b0,efficiency,diffusion,velocity,dm0,db1,db1_phase,db0,defficiency,ddiffusion,dvelocity,duration,kind,flip,phase,action,output_index,shim_index,saturation,rf_frequency,dduration,dflip,dphase,table,dtable,pool_index,profile,profile_index,lineshape,pairs,pair_index,dpairs,output_real,output_imag,trajectory,base,atom_count,event_count,output_count,state_count,rows,flow_scale,washout_scale,profile_step,lineshape_step,locations,profile_bins,lineshape_bins,n,m,blocks,planes,atom_stride,shimmed,profiled,dynamic,directed_pairs,directed_table,following,keep,off_axis,moving,diffusing,transmit,density,inverting,P,S", "ppppppppppppppppppppppppppppppppppppppiiiiiiffffiiiiiiiiiiiiiiiiiiiiiii", 70, 69, 69, BLOCHSIM_LANES__pooled_kernel},
Expand Down
4 changes: 2 additions & 2 deletions src/blochsim/_layout.cu
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ layout_real::Params real_params(const Reader& r) {
p.output_index = r.ints("output_index");
p.shim_index = r.ints("shim_index");
p.action = static_cast<const unsigned char*>(r.a[r.at("action")].p);
p.output_real = r.outputs("output_real");
p.output_imag = r.outputs("output_imag");
p.output = r.outputs("output");
p.sample_factor = r.floats("sample_factor");
p.atom_count = static_cast<int>(r.integer("atom_count"));
p.train_count = static_cast<int>(r.integer("train_count"));
p.event_count = static_cast<int>(r.integer("event_count"));
Expand Down
4 changes: 3 additions & 1 deletion src/blochsim/_layout_real.cu
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
namespace blochsim_layout {
namespace {

// Held to the registers that let eight programs share an SM; uncapped, nvcc
// spends more and fewer programs hide each other's latency.
template <class T, bool MAPS, int Y, bool ONE_TRAIN>
__global__ void __launch_bounds__(32 * WARPS) real_kernel(layout_real::Params p) {
__global__ void __launch_bounds__(32 * WARPS, 8) real_kernel(layout_real::Params p) {
layout_real::real_loop<T, MAPS, Y, ONE_TRAIN>(p);
}

Expand Down
12 changes: 9 additions & 3 deletions src/blochsim/_layout_real.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@ struct Params {
const float *d_t1, *d_t2, *d_m0, *d_b1, *d_inversion_efficiency, *d_diffusion, *d_duration, *d_flip;
const int *kind, *output_index, *shim_index;
const unsigned char* action;
float *output_real, *output_imag;
// Each sample as a complex number, real then imaginary; and the complex
// factor each sample is multiplied by, or null for none.
float* output;
const float* sample_factor;
int atom_count, train_count, event_count, output_count, state_count, width;
bool single_train, atom_stride, shimmed, diffusing, transmit, density, inverting;
};
Expand Down Expand Up @@ -123,12 +126,15 @@ __device__ __forceinline__ void real_loop(const Params& p) {
float held[Y];
int run_start = 0, run_len = 0;
auto flush = [&]() {
// The signal is i times what a thread holds, before its factor.
const float2 factor = p.sample_factor && state < run_len
? reinterpret_cast<const float2*>(p.sample_factor)[run_start + state]
: make_float2(1.0f, 0.0f);
#pragma unroll
for (int y = 0; y < Y; ++y) {
if (state < run_len && active[y]) {
const long at_out = static_cast<long>(problem[y]) * p.output_count + run_start + state;
p.output_real[at_out] = 0.0f;
p.output_imag[at_out] = held[y];
reinterpret_cast<float2*>(p.output)[at_out] = make_float2(-factor.y * held[y], factor.x * held[y]);
}
}
run_len = 0;
Expand Down
25 changes: 16 additions & 9 deletions src/blochsim/_model/_signal.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,16 +229,23 @@ def along(*inputs: torch.Tensor) -> torch.Tensor:
return self._shaped(self.evaluate(declared, **sequence), batch)

forward_mode()
columns = []
signal = None
for name in names:
tangents = tuple(
torch.ones_like(value) if other == name else torch.zeros_like(value)
for other, value in zip(names, primals, strict=True)
# One direction per property, batched: the signal is simulated once and
# only the directional derivatives are taken per property.
directions = tuple(
torch.stack(
[
torch.ones_like(value) if other == name else torch.zeros_like(value)
for name in names
]
)
signal, column = torch.func.jvp(along, primals, tangents)
columns.append(column)
jacobian = columns[0] if isinstance(diff, str) else torch.stack(columns, dim=-2)
for other, value in zip(names, primals, strict=True)
)

def column(*tangents: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return torch.func.jvp(along, primals, tangents)

signal, columns = torch.func.vmap(column, out_dims=(None, 0))(*directions)
jacobian = columns[0] if isinstance(diff, str) else columns.movedim(0, -2)
return like(signal, backend), like(jacobian, backend)

# -- what a subclass does not have to write -----------------------------
Expand Down
4 changes: 4 additions & 0 deletions src/blochsim/_model/_state_machine.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,9 @@ class Simulator(_SignalModel):
across_slice: Any = None
crusher_dephasing_rad: float = 0.0
voxel_size_m: float | None = None
# The constant every recorded sample is multiplied by: the receiver phase
# that puts a sequence's signal on the axis its readers expect.
receiver: complex = 1.0

# Every attribute the constructor sets has a value here as well, so a
# closed form that writes an __init__ of its own -- taking the two blocks
Expand Down Expand Up @@ -1006,6 +1009,7 @@ def evaluate(self, properties: Mapping[str, Any], **sequence: Any) -> torch.Tens
nstates=states,
events=events,
across_slice=profile,
_receiver=self.receiver,
**settings,
)
.signal
Expand Down
Loading
Loading