audiosampler: keep the filter and LFO state out of the voice table

docs/guides/play-recorded-notes.md

@@ -85,6 +85,13 @@ on an ESP32-P4, where 48 filtered voices fit a 256-frame block and 64 only
 just do: see
 [what the filter and the LFOs cost](../how-it-works/upstream-diff.md#what-the-filter-and-the-lfos-cost).
 
+They cost no memory until a voice uses them. The first `play()` that asks for
+any of them, or the first `cutoff` or nonzero `vibrato_depth` set on a voice,
+gives that voice about 270 bytes for their state, which it then keeps. A
+`Sampler(voice_count=32)` on MicroPython takes about 19 KB whatever its notes
+ask for, and one whose 32 voices all use them takes about 7 KB more, in 32
+small pieces rather than one large one.
+
 ## How it renders
 
 Every voice is mixed in one C call per block, summed in float and quantised

src/audiosampler/SamplerVoice.c

@@ -21,9 +21,24 @@ mp_obj_t audiosampler_samplervoice_new(audiosampler_sampler_obj_t *parent,
     self->pitch = mp_obj_new_float(1.0);
     self->cutoff = mp_const_none;
     self->vibrato_depth = mp_obj_new_float(0.0);
+    self->mod = NULL;
     return MP_OBJ_FROM_PTR(self);
 }
 
+// Gives the voice its filter, envelope and LFO state the first time it needs
+// any of them, so a sampler whose notes never ask carries none. Allocated
+// outside the pump lock, since an allocation may collect.
+static void samplervoice_need_mod(audiosampler_samplervoice_obj_t *self) {
+    if (self->mod != NULL) {
+        return;
+    }
+    audiodsp_sampler_mod_t *pair = m_malloc(2 * sizeof(audiodsp_sampler_mod_t));
+    audiodsp_pump_lock_acquire();
+    audiodsp_sampler_attach_mod(&self->parent->sampler, self->index, pair);
+    self->mod = pair;
+    audiodsp_pump_lock_release();
+}
+
 static float float_arg(mp_obj_t value, float fallback) {
     return value == mp_const_none ? fallback : (float)mp_obj_get_float(value);
 }
@@ -128,6 +143,10 @@ static mp_obj_t audiosampler_samplervoice_play(size_t n_args,
     note.lfo_to_cutoff = float_arg(args[ARG_lfo_to_cutoff].u_obj, 0.0f);
     note.lfo_to_volume = float_arg(args[ARG_lfo_to_volume].u_obj, 0.0f);
 
+    if (audiodsp_sampler_note_modulated(&note)) {
+        samplervoice_need_mod(self);
+    }
+
     // Read and checked first, stored under the lock: a pull landing between
     // the kernel's play() and the reference store would read a buffer the
     // GC may already be free to take.
@@ -293,6 +312,9 @@ static mp_obj_t audiosampler_samplervoice_obj_set_cutoff(mp_obj_t self_in,
     audiosample_check_for_deinit(&self->parent->base);
     float cutoff = value == mp_const_none ? 0.0f
         : (float)mp_obj_get_float(value);
+    if (cutoff > 0.0f) {
+        samplervoice_need_mod(self);
+    }
     audiodsp_pump_lock_acquire();
     audiodsp_sampler_set_cutoff(&self->parent->sampler, self->index, cutoff);
     self->cutoff = value;
@@ -320,6 +342,9 @@ static mp_obj_t audiosampler_samplervoice_obj_set_vibrato_depth(
     audiosampler_samplervoice_obj_t *self = MP_OBJ_TO_PTR(self_in);
     audiosample_check_for_deinit(&self->parent->base);
     float cents = (float)mp_obj_get_float(value);
+    if (cents != 0.0f) {
+        samplervoice_need_mod(self);
+    }
     audiodsp_pump_lock_acquire();
     audiodsp_sampler_set_vibrato_depth(&self->parent->sampler, self->index,
         cents);

src/audiosampler/SamplerVoice.h

@@ -29,6 +29,10 @@ typedef struct {
     mp_obj_t pitch;
     mp_obj_t cutoff;
     mp_obj_t vibrato_depth;
+    // The voice's and its tail's filter, envelope and LFO state, allocated
+    // the first time a note or a live change asks for any of them, and held
+    // here as well as in the voice table so the GC keeps it.
+    audiodsp_sampler_mod_t *mod;
 } audiosampler_samplervoice_obj_t;
 
 extern const mp_obj_type_t audiosampler_samplervoice_type;

src/circuitpython_spike/shared-bindings/audiosampler/SamplerVoice.c

@@ -28,9 +28,21 @@ mp_obj_t audiosampler_samplervoice_new(audiosampler_sampler_obj_t *parent,
     self->pitch = mp_obj_new_float(1.0);
     self->cutoff = mp_const_none;
     self->vibrato_depth = mp_obj_new_float(0.0);
+    self->mod = NULL;
     return MP_OBJ_FROM_PTR(self);
 }
 
+// Gives the voice its filter, envelope and LFO state the first time it needs
+// any of them, so a sampler whose notes never ask carries none.
+static void samplervoice_need_mod(audiosampler_samplervoice_obj_t *self) {
+    if (self->mod != NULL) {
+        return;
+    }
+    audiodsp_sampler_mod_t *pair = m_malloc(2 * sizeof(audiodsp_sampler_mod_t));
+    audiodsp_sampler_attach_mod(&self->parent->sampler, self->index, pair);
+    self->mod = pair;
+}
+
 static float float_arg(mp_obj_t value, float fallback) {
     return value == mp_const_none ? fallback : (float)mp_obj_get_float(value);
 }
@@ -187,6 +199,9 @@ static mp_obj_t audiosampler_samplervoice_play(size_t n_args,
     note.lfo_to_pitch = float_arg(args[ARG_lfo_to_pitch].u_obj, 0.0f);
     note.lfo_to_cutoff = float_arg(args[ARG_lfo_to_cutoff].u_obj, 0.0f);
     note.lfo_to_volume = float_arg(args[ARG_lfo_to_volume].u_obj, 0.0f);
+    if (audiodsp_sampler_note_modulated(&note)) {
+        samplervoice_need_mod(self);
+    }
 
     bool handed = audiodsp_sampler_audible(&self->parent->sampler, self->index);
     if (!audiodsp_sampler_play(&self->parent->sampler, self->index,
@@ -353,6 +368,9 @@ static mp_obj_t audiosampler_samplervoice_obj_set_cutoff(mp_obj_t self_in,
     audiosample_check_for_deinit(&self->parent->base);
     float cutoff = value == mp_const_none ? 0.0f
         : (float)mp_obj_get_float(value);
+    if (cutoff > 0.0f) {
+        samplervoice_need_mod(self);
+    }
     audiodsp_sampler_set_cutoff(&self->parent->sampler, self->index, cutoff);
     self->cutoff = value;
     return mp_const_none;
@@ -380,6 +398,9 @@ static mp_obj_t audiosampler_samplervoice_obj_set_vibrato_depth(
     audiosampler_samplervoice_obj_t *self = MP_OBJ_TO_PTR(self_in);
     audiosample_check_for_deinit(&self->parent->base);
     float cents = (float)mp_obj_get_float(value);
+    if (cents != 0.0f) {
+        samplervoice_need_mod(self);
+    }
     audiodsp_sampler_set_vibrato_depth(&self->parent->sampler, self->index,
         cents);
     self->vibrato_depth = value;

src/circuitpython_spike/shared-module/audiosampler/Sampler.h

@@ -47,6 +47,10 @@ typedef struct {
     mp_obj_t pitch;
     mp_obj_t cutoff;
     mp_obj_t vibrato_depth;
+    // The voice's and its tail's filter, envelope and LFO state, allocated
+    // the first time a note or a live change asks for any of them, and held
+    // here as well as in the voice table so the GC keeps it.
+    audiodsp_sampler_mod_t *mod;
 } audiosampler_samplervoice_obj_t;
 
 void audiosampler_sampler_reset_buffer(audiosampler_sampler_obj_t *self,

src/cpython/_audiodsp.c

@@ -2882,6 +2882,32 @@ static void sampler_state_hand(audiodsp_sampler_object_t *self, uint32_t index)
     }
 }
 
+// Each voice's filter, envelope and LFO state, with its tail's, is a pair
+// allocated the first time a note or a live change on the voice asks for any
+// of them; the voice table holds the pointer.
+static int sampler_state_need_mod(audiodsp_sampler_object_t *self,
+    uint32_t index) {
+    if (audiodsp_sampler_has_mod(&self->sampler, index)) {
+        return 0;
+    }
+    audiodsp_sampler_mod_t *pair = PyMem_Calloc(2,
+        sizeof(audiodsp_sampler_mod_t));
+    if (pair == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    audiodsp_sampler_attach_mod(&self->sampler, index, pair);
+    return 0;
+}
+
+static void sampler_state_free_mods(audiodsp_sampler_object_t *self) {
+    for (uint32_t i = 0; self->voices != NULL &&
+         i < self->sampler.voice_count; ++i) {
+        PyMem_Free(self->voices[i].mod);
+        self->voices[i].mod = NULL;
+    }
+}
+
 static int sampler_state_init(audiodsp_sampler_object_t *self, PyObject *args,
     PyObject *kwargs) {
     static char *keywords[] = {"sample_rate", "channel_count", "frames",
@@ -2911,6 +2937,7 @@ static int sampler_state_init(audiodsp_sampler_object_t *self, PyObject *args,
         return -1;
     }
     sampler_state_drop_all(self);
+    sampler_state_free_mods(self);
     PyMem_Free(self->voices);
     PyMem_Free(self->mix);
     PyMem_Free(self->out);
@@ -2935,6 +2962,7 @@ static int sampler_state_init(audiodsp_sampler_object_t *self, PyObject *args,
 static void sampler_state_dealloc(audiodsp_sampler_object_t *self) {
     PyTypeObject *type = Py_TYPE(self);
     sampler_state_drop_all(self);
+    sampler_state_free_mods(self);
     PyMem_Free(self->voices);
     PyMem_Free(self->mix);
     PyMem_Free(self->out);
@@ -3053,6 +3081,11 @@ static PyObject *sampler_state_play(audiodsp_sampler_object_t *self,
     note.lfo_to_pitch = (float)mod[15];
     note.lfo_to_cutoff = (float)mod[16];
     note.lfo_to_volume = (float)mod[17];
+    if (audiodsp_sampler_note_modulated(&note) &&
+        sampler_state_need_mod(self, index) < 0) {
+        PyBuffer_Release(&view);
+        return NULL;
+    }
     // Let go of what this voice was reading only once the new buffer is
     // known to be good, so a refused play() leaves the voice as it was.
     // The kernel leaves the voice and its tail untouched on a refusal.
@@ -3155,6 +3188,9 @@ static PyObject *sampler_state_set_cutoff(audiodsp_sampler_object_t *self,
     if (!PyArg_ParseTuple(args, "Od:set_cutoff", &index_obj, &value)) return NULL;
     uint32_t index;
     if (sampler_state_voice(self, index_obj, &index) < 0) return NULL;
+    if ((float)value > 0.0f && sampler_state_need_mod(self, index) < 0) {
+        return NULL;
+    }
     audiodsp_sampler_set_cutoff(&self->sampler, index, (float)value);
     Py_RETURN_NONE;
 }
@@ -3168,6 +3204,9 @@ static PyObject *sampler_state_set_vibrato_depth(
     }
     uint32_t index;
     if (sampler_state_voice(self, index_obj, &index) < 0) return NULL;
+    if ((float)value != 0.0f && sampler_state_need_mod(self, index) < 0) {
+        return NULL;
+    }
     audiodsp_sampler_set_vibrato_depth(&self->sampler, index, (float)value);
     Py_RETURN_NONE;
 }