Build as a CircuitPython 11 user C module; survive a reload; never spin a board off USB

.gitignore

@@ -12,3 +12,6 @@ dist/
 # Cursor IDE throwaways (do not ignore the whole .cursor/ tree)
 .cursor/debug*
 .cursor/plans/
+
+# lvgl-bindings fetched by scripts/fetch_bindings.sh for a clone on its own
+.deps/

LVGL_BINDINGS_COMMIT

@@ -1 +1 @@
-7316c48517940b9481307ef8cca9d80e591f9b00
+2858c42a4f3e2936cfd7674ac86ab92603007850

README.md

@@ -13,6 +13,28 @@ Requires sibling clones of [lvgl-bindings](https://github.com/PyDevices/lvgl-bin
 
 **`display_driver` is not here.** LVGL's PyDevices coordinator lives in [pydevices `lib/`](https://github.com/PyDevices/pydevices/blob/main/lib/display_driver.py) and comes with `pydevices`, beside the `appdev`, `events`, `keys` and `multimer` it needs. On CircuitPython, install pydevices as source (`--no-mpy`) onto `CIRCUITPY`.
 
+## Build it as a user C module
+
+CircuitPython 11 builds user C modules the way MicroPython does, so you can
+hand this directory to any port's `make` without patching CircuitPython:
+
+```bash
+make -C circuitpython/ports/unix USER_C_MODULES=/path/to/lvgl-circuitpython
+make -C circuitpython/ports/raspberrypi BOARD=<board> CIRCUITPY_GIFIO=0 \
+    USER_C_MODULES=/path/to/lvgl-circuitpython
+```
+
+`CIRCUITPY_GIFIO=0` is for boards: LVGL links its own AnimatedGIF decoder,
+which collides with `gifio`'s. A sibling `lvgl-bindings/` checkout at the
+commit in `LVGL_BINDINGS_COMMIT` is used if there is one; otherwise the build
+fetches that commit into `.deps/`. LVGL adds about 1 MB of firmware, more than
+most boards' default firmware region holds, and its qstrs need CircuitPython's
+four-byte qstr offsets (`MICROPY_QSTR_OFFSET_BYTES`, set by `micropython.mk`)
+where a port has them. The glue is [micropython.mk](micropython.mk).
+
+The patch-script route below (`apply_cp_patches.sh`, for CircuitPython 10.x)
+still works.
+
 ## Workspace layout
 
 Place this repo as a sibling of `lvgl-bindings/` and `circuitpython/`:

circuitpython.mk

@@ -40,7 +40,8 @@ LV_CP_LVGL_SOURCES := $(shell find $(LVGL_DIR)/src -type f -name '*.c')
 # CIRCUITPY_GIFIO=0 when CIRCUITPY_LVGL=1 so LVGL's libs/gif/gif.c (LV_USE_GIF)
 # can link. No lvgl-bindings generator change; constraint is build-side.
 LV_CP_SOURCES := $(LV_CP_MOD_DIR)/src/lv_mem_core_circuitpython.c \
-	$(LV_CP_MOD_DIR)/src/lv_jpegio_decoder_circuitpython.c
+	$(LV_CP_MOD_DIR)/src/lv_jpegio_decoder_circuitpython.c \
+	$(LV_CP_MOD_DIR)/src/lv_assert_circuitpython.c
 
 ifeq ($(wildcard $(LVCP_C)),)
 $(error $(LVCP_C) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython)

micropython.mk

@@ -0,0 +1,131 @@
+# CircuitPython user C module glue for LVGL and its generated bindings.
+#
+# CircuitPython builds user C modules the way MicroPython does, so this
+# directory can be handed to any CircuitPython port's make as is, with no
+# patches to CircuitPython's own tree:
+#
+#   make -C ports/unix USER_C_MODULES=/path/to/lvgl-circuitpython
+#   make -C ports/raspberrypi BOARD=adafruit_feather_rp2040 CIRCUITPY_GIFIO=0 \
+#       USER_C_MODULES=/path/to/lvgl-circuitpython
+#
+# On a board, CIRCUITPY_GIFIO=0 goes on the make line: LVGL links its own
+# copy of the AnimatedGIF decoder (LV_USE_GIF in lvgl-bindings' lv_conf.h),
+# and CircuitPython's gifio links another, so the two collide at link time.
+# The unix port compiles no gifio and needs nothing.
+#
+# apply_cp_patches.sh and circuitpython.mk are the older route (they copy the
+# module into CircuitPython's tree and patch its makefiles); this file does
+# not use them.
+
+LVCP_DIR := $(USERMOD_DIR)
+
+ifeq ($(wildcard $(TOP)/py/circuitpy_mpconfig.h),)
+$(error lvgl-circuitpython is for CircuitPython builds; MicroPython builds use lvgl-micropython)
+endif
+
+ifeq ($(CIRCUITPY_GIFIO),1)
+$(error lvgl-circuitpython: add CIRCUITPY_GIFIO=0 to the make line. LVGL links its own AnimatedGIF decoder, which collides with gifio's)
+endif
+
+# lvgl-bindings: a sibling checkout wins; without one (a clone on its own),
+# the pinned commit is fetched into .deps/, only what this module compiles.
+LV_BINDINGS_DIR ?= $(abspath $(LVCP_DIR)/../lvgl-bindings)
+ifeq ($(wildcard $(LV_BINDINGS_DIR)/lv_conf.h),)
+LV_BINDINGS_DIR := $(abspath $(LVCP_DIR)/.deps/lvgl-bindings)
+LVCP_FETCH := $(shell bash $(LVCP_DIR)/scripts/fetch_bindings.sh 1>&2; echo $$?)
+ifeq ($(wildcard $(LV_BINDINGS_DIR)/lv_conf.h),)
+$(error lvgl-circuitpython: fetching lvgl-bindings failed (scripts/fetch_bindings.sh exit $(LVCP_FETCH)); set LV_BINDINGS_DIR to a checkout of the commit in LVGL_BINDINGS_COMMIT)
+endif
+endif
+
+LVGL_DIR := $(LV_BINDINGS_DIR)/lvgl
+LVCP_C := $(LV_BINDINGS_DIR)/generated/lvgl_circuitpython.c
+LVCP_H := $(LV_BINDINGS_DIR)/generated/lvgl_circuitpython.h
+LV_BINDINGS_PIN := $(strip $(shell cat $(LVCP_DIR)/LVGL_BINDINGS_COMMIT 2>/dev/null))
+LV_BINDINGS_DIRTY := $(shell \
+	git -C $(LV_BINDINGS_DIR) cat-file -e $(LV_BINDINGS_PIN)^{commit} 2>/dev/null && \
+	git -C $(LV_BINDINGS_DIR) diff --quiet $(LV_BINDINGS_PIN) -- generated/lvgl_circuitpython.c generated/lvgl_circuitpython.h lvgl lv_conf.h && \
+	git -C $(LV_BINDINGS_DIR) diff --quiet -- generated/lvgl_circuitpython.c generated/lvgl_circuitpython.h lvgl lv_conf.h || echo 1)
+
+ifeq ($(LV_BINDINGS_PIN),)
+$(error Missing $(LVCP_DIR)/LVGL_BINDINGS_COMMIT)
+endif
+ifneq ($(LV_BINDINGS_DIRTY),)
+$(error $(LV_BINDINGS_DIR) does not match pinned binding inputs $(LV_BINDINGS_PIN); check out that commit or run scripts/sync_from_lvgl_bindings.sh with an exact ref)
+endif
+$(if $(wildcard $(LVCP_C)),,$(error $(LVCP_C) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython))
+$(if $(wildcard $(LVCP_H)),,$(error $(LVCP_H) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython))
+
+# LVGL itself. Its host backends (SDL, X11, Wayland, OpenGL ES, ...) need host
+# libraries and are left out of board builds; on unix each compiles to nothing
+# unless lv_conf.h turns it on.
+LVCP_LVGL_SOURCES := $(shell find $(LVGL_DIR)/src -type f -name '*.c')
+ifeq ($(findstring /ports/unix,$(abspath $(CURDIR))),)
+LVCP_EXCLUDE_DIRS := \
+	$(LVGL_DIR)/src/drivers/opengles \
+	$(LVGL_DIR)/src/drivers/sdl \
+	$(LVGL_DIR)/src/drivers/glfw \
+	$(LVGL_DIR)/src/drivers/x11 \
+	$(LVGL_DIR)/src/drivers/wayland \
+	$(LVGL_DIR)/src/drivers/evdev \
+	$(LVGL_DIR)/src/drivers/libinput \
+	$(LVGL_DIR)/src/drivers/qnx \
+	$(LVGL_DIR)/src/drivers/uefi \
+	$(LVGL_DIR)/src/drivers/nuttx \
+	$(LVGL_DIR)/src/drivers/windows \
+	$(LVGL_DIR)/src/draw/opengles \
+	$(LVGL_DIR)/src/libs/gltf
+LVCP_LVGL_SOURCES := $(foreach s,$(LVCP_LVGL_SOURCES),$(if $(strip $(foreach d,$(LVCP_EXCLUDE_DIRS),$(findstring $(d)/,$(s)))),,$(s)))
+endif
+
+# LVGL has no MP_QSTR_* in it, so it goes in SRC_USERMOD_LIB_C, which py.mk
+# compiles and links but never scans for qstrs. Scanning it would only cost
+# time, and on espressif the scan's command line passes the kernel's limit.
+SRC_USERMOD_LIB_C += $(LVCP_LVGL_SOURCES)
+
+# The CircuitPython side: the module (registered with MP_REGISTER_MODULE in
+# shared-bindings/lvgl/__init__.c), the generated bindings, LVGL's allocator on
+# CircuitPython's heap, LVGL's JPEG decoder on CircuitPython's TJpgDec (a no-op
+# without jpegio), and what an LVGL assertion does (safe mode, not a hang).
+# These are scanned for qstrs.
+LVCP_SPIKE := $(LVCP_DIR)/src/circuitpython_spike
+LVCP_SOURCES := \
+	$(LVCP_SPIKE)/shared-bindings/lvgl/__init__.c \
+	$(LVCP_SPIKE)/shared-module/lvgl/__init__.c \
+	$(LVCP_DIR)/src/lv_mem_core_circuitpython.c \
+	$(LVCP_DIR)/src/lv_jpegio_decoder_circuitpython.c \
+	$(LVCP_DIR)/src/lv_assert_circuitpython.c \
+	$(LVCP_C)
+SRC_USERMOD_C += $(LVCP_SOURCES)
+
+# The spike includes "shared-bindings/lvgl/__init__.h"; CircuitPython's own
+# -I$(TOP) comes first and has no lvgl/, so these resolve to the spike's.
+# LV_CIRCUITPYTHON_BUILD picks the allocator above in lv_conf.h and leaves
+# lvgl.init()/deinit() to the spike.
+CFLAGS_USERMOD += -DLV_CIRCUITPYTHON_BUILD=1 -I$(LVCP_SPIKE) -I$(LV_BINDINGS_DIR) -I$(LVGL_DIR) -Wno-unused-function
+
+# LVGL's bindings bring about 53 KB of qstrs. CircuitPython 11 keeps a ROM qstr
+# pool's strings in one blob addressed by 16-bit offsets, 64 KB at most, which
+# a board's own modules and LVGL together pass. Where CircuitPython has the
+# MICROPY_QSTR_OFFSET_BYTES setting, four-byte offsets lift that limit for
+# about two bytes of flash per qstr.
+CFLAGS_USERMOD += -DMICROPY_QSTR_OFFSET_BYTES=4
+
+# Upstream LVGL and the generated bindings trip warnings CircuitPython's ports
+# make errors of, and the board ports append those -Werror flags after
+# CFLAGS_USERMOD, so a module-wide -Wno-* would lose. A target-specific flag
+# lands last, so they are turned off per object, for this module's objects
+# only. -Wfloat-equal, -Wdouble-promotion and -Wcast-align are the board
+# ports' (arc and chart compare floats directly); the rest are LVGL's style.
+LVCP_OBJ_CFLAGS := -Wno-cast-align -Wno-nested-externs -Wno-unused-parameter \
+	-Wno-sign-compare -Wno-missing-prototypes -Wno-old-style-definition \
+	-Wno-float-conversion -Wno-double-promotion -Wno-shadow -Wno-type-limits \
+	-Wno-suggest-attribute=format -Wno-float-equal -Wno-unused-const-variable \
+	-Wno-unused-but-set-variable -Wno-unused-function -Wno-undef
+
+# py.mk names a user module's object $(BUILD)/<module dir name>/<path inside
+# it>.o, and a source outside the module (lvgl-bindings beside it)
+# $(BUILD)/<absolute path>.o. Both spellings are declared.
+LVCP_OBJ = $(BUILD)/$(patsubst $(LVCP_DIR)/%,$(notdir $(LVCP_DIR))/%,$(1)) $(BUILD)/$(1)
+$(foreach _s,$(LVCP_LVGL_SOURCES) $(LVCP_SOURCES),\
+	$(eval $(call LVCP_OBJ,$(_s:.c=.o)): CFLAGS += $(LVCP_OBJ_CFLAGS)))

scripts/fetch_bindings.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Fetch lvgl-bindings at the commit in LVGL_BINDINGS_COMMIT into
+# .deps/lvgl-bindings, with only what this module compiles: generated/,
+# lv_conf.h, and LVGL's src/ plus its top-level headers. Shallow, sparse and
+# blobless, so it downloads tens of MB rather than LVGL's whole history.
+#
+# micropython.mk runs this itself when there is no LV_BINDINGS_DIR and no
+# lvgl-bindings checkout beside this one (a clone on its own); with a
+# sibling checkout the sibling wins and this never runs.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")/.." && pwd)
+DEST="$HERE/.deps/lvgl-bindings"
+URL="${LV_BINDINGS_REPO:-https://github.com/PyDevices/lvgl-bindings.git}"
+PIN=$(tr -d '[:space:]' < "$HERE/LVGL_BINDINGS_COMMIT")
+
+if [[ -d "$DEST/.git" && "$(git -C "$DEST" rev-parse HEAD 2>/dev/null)" == "$PIN" && -f "$DEST/lvgl/lvgl.h" ]]; then
+    echo "lvgl-bindings ${PIN:0:12} already in .deps"
+    exit 0
+fi
+rm -rf "$DEST"
+mkdir -p "$DEST"
+git -C "$DEST" init -q
+git -C "$DEST" remote add origin "$URL"
+git -C "$DEST" config extensions.partialClone origin
+git -C "$DEST" sparse-checkout set generated lvgl
+git -C "$DEST" fetch -q --depth 1 --filter=blob:none origin "$PIN"
+git -C "$DEST" checkout -q FETCH_HEAD
+# LVGL itself, as its own partial clone at the commit lvgl-bindings records:
+# `git submodule update` ignores --filter and pulls a ~100 MB pack.
+LVGL_SHA=$(git -C "$DEST" ls-tree HEAD lvgl | awk '{print $3}')
+LVGL_URL=$(git -C "$DEST" config -f .gitmodules submodule.lvgl.url)
+rm -rf "$DEST/lvgl"
+mkdir -p "$DEST/lvgl"
+git -C "$DEST/lvgl" init -q
+git -C "$DEST/lvgl" remote add origin "$LVGL_URL"
+git -C "$DEST/lvgl" config extensions.partialClone origin
+git -C "$DEST/lvgl" sparse-checkout set src
+git -C "$DEST/lvgl" fetch -q --depth 1 --filter=blob:none origin "$LVGL_SHA"
+git -C "$DEST/lvgl" checkout -q FETCH_HEAD
+echo "lvgl-bindings ${PIN:0:12} fetched into $DEST"

src/circuitpython_spike/shared-bindings/lvgl/__init__.c

@@ -24,6 +24,38 @@ MP_DEFINE_CONST_FUN_OBJ_0(lvgl_deinit_obj, lvgl_module_deinit_fn);
 
 MP_REGISTER_MODULE(MP_QSTR_lvgl, lvgl_module);
 
+#if MICROPY_MODULE_BUILTIN_INIT && MICROPY_MODULE_ATTR_DELEGATION
+// CircuitPython starts a new VM on every reload (a saved file, Ctrl-D,
+// supervisor.reload()) and does not clear root pointers between VMs. LVGL's
+// state, lv_global_t and everything it reaches, lives on the GC heap behind
+// them, so in the next VM they point into a heap that has been wiped:
+// lv.is_initialized() reads garbage and the first display_create asks for
+// megabytes. A built-in module's __init__ runs whenever the import system
+// finds it among the built-ins; the generated module dict has none, so this
+// delegation answers the lookup and drops the stale state. Built-ins are not
+// kept in sys.modules, so __init__ would otherwise run on every
+// `import lvgl` and drop live state too: it puts lvgl in sys.modules, which a
+// later import in the same VM finds first, and which a new VM starts without.
+// (Deleting it from sys.modules, or importing with an empty sys.path, drops
+// live state; LVGL is then initialised afresh.)
+static mp_obj_t lvgl_module_vm_init(void) {
+    mp_lv_deinit_gc();
+    mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)),
+        MP_OBJ_NEW_QSTR(MP_QSTR_lvgl), MP_OBJ_FROM_PTR(&lvgl_module));
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_0(lvgl_module_vm_init_obj, lvgl_module_vm_init);
+
+void lvgl_module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
+void lvgl_module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
+    (void)self_in;
+    if (dest[0] == MP_OBJ_NULL && attr == MP_QSTR___init__) {
+        dest[0] = MP_OBJ_FROM_PTR(&lvgl_module_vm_init_obj);
+    }
+}
+MP_REGISTER_MODULE_DELEGATION(lvgl_module, lvgl_module_attr);
+#endif
+
 void lvgl_init(void) {
     shared_modules_lvgl_init();
 }

src/lv_assert_circuitpython.c

@@ -0,0 +1,29 @@
+/**
+ * @file lv_assert_circuitpython.c
+ *
+ * What an LVGL assertion does on CircuitPython (lvgl-bindings' lv_conf.h
+ * names this function as LV_ASSERT_HANDLER when LV_CIRCUITPYTHON_BUILD is
+ * set). LVGL's default is to spin forever, and a board spinning in C stops
+ * answering USB: its drive and serial port vanish until someone presses reset.
+ * Most LVGL assertions on a board are an allocation that failed because the
+ * heap is full, so instead the board reboots into safe mode, which reports
+ * "Third-party firmware fatal error." and keeps CIRCUITPY reachable to fix
+ * code.py. On the unix port the process aborts.
+ */
+#include "py/mpconfig.h"
+
+void lv_circuitpython_assert_failed(void);
+
+#if defined(__unix__) || defined(__APPLE__)
+#include <stdlib.h>
+
+void lv_circuitpython_assert_failed(void) {
+    abort();
+}
+#else
+#include "supervisor/shared/safe_mode.h"
+
+void lv_circuitpython_assert_failed(void) {
+    reset_into_safe_mode(SAFE_MODE_SDK_FATAL_ERROR);
+}
+#endif

src/lv_mem_core_circuitpython.c

@@ -70,7 +70,12 @@ void *lv_malloc_core(size_t size)
      * dereferencing a bogus mp_obj type pointer and crashing (SIGSEGV). */
     return gc_alloc(size, false);
 #else
-    return m_malloc(size);
+    /* The _maybe allocators return NULL when the heap is full; m_malloc would
+     * raise MemoryError, a longjmp straight out of LVGL's C that leaves a
+     * refresh half done (its flushing and rendering flags set) for the next
+     * pass to spin on. With NULL, LVGL fails the way it was written to:
+     * LV_ASSERT_MALLOC, which lv_assert_circuitpython.c turns into safe mode. */
+    return m_malloc_maybe(size);
 #endif
 }
 
@@ -79,7 +84,7 @@ void *lv_realloc_core(void *p, size_t new_size)
 #if MICROPY_MALLOC_USES_ALLOCATED_SIZE
     return gc_realloc(p, new_size, true);
 #else
-    return m_realloc(p, new_size);
+    return m_realloc_maybe(p, new_size, true);
 #endif
 }
 

tests/test_integration.py

@@ -62,3 +62,39 @@ def test_jpegio_decoder_shim_is_wired():
     assert "file_has_soi" in text
     assert "lv_fs_get_ext" not in text
     assert "has_jpeg_ext" not in text
+
+
+def test_user_c_module_keeps_lvgl_out_of_the_qstr_scan():
+    make = (ROOT / "micropython.mk").read_text()
+    # LVGL has no MP_QSTR_*: compiled and linked, never scanned.
+    assert "SRC_USERMOD_LIB_C += $(LVCP_LVGL_SOURCES)" in make
+    assert "SRC_USERMOD_C += $(LVCP_SOURCES)" in make
+    assert "generated/lvgl_circuitpython.c" in make
+    assert "LVGL_BINDINGS_COMMIT" in make
+    # gifio's AnimatedGIF collides with LVGL's at link time: stop early.
+    assert "ifeq ($(CIRCUITPY_GIFIO),1)" in make
+    # The board ports' late -Werror flags lose to per-object flags only.
+    assert "CFLAGS += $(LVCP_OBJ_CFLAGS)" in make
+
+
+def test_a_reload_drops_lvgl_state_from_the_previous_vm():
+    shared_bindings = (
+        ROOT / "src/circuitpython_spike/shared-bindings/lvgl/__init__.c"
+    ).read_text()
+    assert "MP_REGISTER_MODULE_DELEGATION(lvgl_module, lvgl_module_attr);" in shared_bindings
+    assert "attr == MP_QSTR___init__" in shared_bindings
+    assert "mp_lv_deinit_gc();" in shared_bindings
+    # Built-ins stay out of sys.modules, so without this every `import lvgl`
+    # would run __init__ again and drop live state.
+    assert "MP_STATE_VM(mp_loaded_modules_dict)" in shared_bindings
+
+
+def test_an_lvgl_failure_never_spins_a_board_off_usb():
+    mem = (ROOT / "src/lv_mem_core_circuitpython.c").read_text()
+    # NULL, not MemoryError: an exception would longjmp out of LVGL's C.
+    assert "m_malloc_maybe(size)" in mem
+    assert "m_realloc_maybe(p, new_size, true)" in mem
+    handler = (ROOT / "src/lv_assert_circuitpython.c").read_text()
+    assert "reset_into_safe_mode(SAFE_MODE_SDK_FATAL_ERROR)" in handler
+    assert "src/lv_assert_circuitpython.c" in (ROOT / "micropython.mk").read_text()
+    assert "src/lv_assert_circuitpython.c" in (ROOT / "circuitpython.mk").read_text()