diff --git a/.gitignore b/.gitignore index a386672..ecd44b5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ dist/ # Cursor IDE throwaways (do not ignore the whole .cursor/ tree) .cursor/debug* .cursor/plans/ + +# lvgl-bindings fetched by scripts/fetch_bindings.sh for a clone on its own +.deps/ diff --git a/LVGL_BINDINGS_COMMIT b/LVGL_BINDINGS_COMMIT index 3d9ca69..7b28ab7 100644 --- a/LVGL_BINDINGS_COMMIT +++ b/LVGL_BINDINGS_COMMIT @@ -1 +1 @@ -7316c48517940b9481307ef8cca9d80e591f9b00 +2858c42a4f3e2936cfd7674ac86ab92603007850 diff --git a/README.md b/README.md index ccf9569..3a2b6f6 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,28 @@ Requires sibling clones of [lvgl-bindings](https://github.com/PyDevices/lvgl-bin **`display_driver` is not here.** LVGL's PyDevices coordinator lives in [pydevices `lib/`](https://github.com/PyDevices/pydevices/blob/main/lib/display_driver.py) and comes with `pydevices`, beside the `appdev`, `events`, `keys` and `multimer` it needs. On CircuitPython, install pydevices as source (`--no-mpy`) onto `CIRCUITPY`. +## Build it as a user C module + +CircuitPython 11 builds user C modules the way MicroPython does, so you can +hand this directory to any port's `make` without patching CircuitPython: + +```bash +make -C circuitpython/ports/unix USER_C_MODULES=/path/to/lvgl-circuitpython +make -C circuitpython/ports/raspberrypi BOARD= CIRCUITPY_GIFIO=0 \ + USER_C_MODULES=/path/to/lvgl-circuitpython +``` + +`CIRCUITPY_GIFIO=0` is for boards: LVGL links its own AnimatedGIF decoder, +which collides with `gifio`'s. A sibling `lvgl-bindings/` checkout at the +commit in `LVGL_BINDINGS_COMMIT` is used if there is one; otherwise the build +fetches that commit into `.deps/`. LVGL adds about 1 MB of firmware, more than +most boards' default firmware region holds, and its qstrs need CircuitPython's +four-byte qstr offsets (`MICROPY_QSTR_OFFSET_BYTES`, set by `micropython.mk`) +where a port has them. The glue is [micropython.mk](micropython.mk). + +The patch-script route below (`apply_cp_patches.sh`, for CircuitPython 10.x) +still works. + ## Workspace layout Place this repo as a sibling of `lvgl-bindings/` and `circuitpython/`: diff --git a/circuitpython.mk b/circuitpython.mk index cfeee70..90b6000 100644 --- a/circuitpython.mk +++ b/circuitpython.mk @@ -40,7 +40,8 @@ LV_CP_LVGL_SOURCES := $(shell find $(LVGL_DIR)/src -type f -name '*.c') # CIRCUITPY_GIFIO=0 when CIRCUITPY_LVGL=1 so LVGL's libs/gif/gif.c (LV_USE_GIF) # can link. No lvgl-bindings generator change; constraint is build-side. LV_CP_SOURCES := $(LV_CP_MOD_DIR)/src/lv_mem_core_circuitpython.c \ - $(LV_CP_MOD_DIR)/src/lv_jpegio_decoder_circuitpython.c + $(LV_CP_MOD_DIR)/src/lv_jpegio_decoder_circuitpython.c \ + $(LV_CP_MOD_DIR)/src/lv_assert_circuitpython.c ifeq ($(wildcard $(LVCP_C)),) $(error $(LVCP_C) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython) diff --git a/micropython.mk b/micropython.mk new file mode 100644 index 0000000..fead8b2 --- /dev/null +++ b/micropython.mk @@ -0,0 +1,131 @@ +# CircuitPython user C module glue for LVGL and its generated bindings. +# +# CircuitPython builds user C modules the way MicroPython does, so this +# directory can be handed to any CircuitPython port's make as is, with no +# patches to CircuitPython's own tree: +# +# make -C ports/unix USER_C_MODULES=/path/to/lvgl-circuitpython +# make -C ports/raspberrypi BOARD=adafruit_feather_rp2040 CIRCUITPY_GIFIO=0 \ +# USER_C_MODULES=/path/to/lvgl-circuitpython +# +# On a board, CIRCUITPY_GIFIO=0 goes on the make line: LVGL links its own +# copy of the AnimatedGIF decoder (LV_USE_GIF in lvgl-bindings' lv_conf.h), +# and CircuitPython's gifio links another, so the two collide at link time. +# The unix port compiles no gifio and needs nothing. +# +# apply_cp_patches.sh and circuitpython.mk are the older route (they copy the +# module into CircuitPython's tree and patch its makefiles); this file does +# not use them. + +LVCP_DIR := $(USERMOD_DIR) + +ifeq ($(wildcard $(TOP)/py/circuitpy_mpconfig.h),) +$(error lvgl-circuitpython is for CircuitPython builds; MicroPython builds use lvgl-micropython) +endif + +ifeq ($(CIRCUITPY_GIFIO),1) +$(error lvgl-circuitpython: add CIRCUITPY_GIFIO=0 to the make line. LVGL links its own AnimatedGIF decoder, which collides with gifio's) +endif + +# lvgl-bindings: a sibling checkout wins; without one (a clone on its own), +# the pinned commit is fetched into .deps/, only what this module compiles. +LV_BINDINGS_DIR ?= $(abspath $(LVCP_DIR)/../lvgl-bindings) +ifeq ($(wildcard $(LV_BINDINGS_DIR)/lv_conf.h),) +LV_BINDINGS_DIR := $(abspath $(LVCP_DIR)/.deps/lvgl-bindings) +LVCP_FETCH := $(shell bash $(LVCP_DIR)/scripts/fetch_bindings.sh 1>&2; echo $$?) +ifeq ($(wildcard $(LV_BINDINGS_DIR)/lv_conf.h),) +$(error lvgl-circuitpython: fetching lvgl-bindings failed (scripts/fetch_bindings.sh exit $(LVCP_FETCH)); set LV_BINDINGS_DIR to a checkout of the commit in LVGL_BINDINGS_COMMIT) +endif +endif + +LVGL_DIR := $(LV_BINDINGS_DIR)/lvgl +LVCP_C := $(LV_BINDINGS_DIR)/generated/lvgl_circuitpython.c +LVCP_H := $(LV_BINDINGS_DIR)/generated/lvgl_circuitpython.h +LV_BINDINGS_PIN := $(strip $(shell cat $(LVCP_DIR)/LVGL_BINDINGS_COMMIT 2>/dev/null)) +LV_BINDINGS_DIRTY := $(shell \ + git -C $(LV_BINDINGS_DIR) cat-file -e $(LV_BINDINGS_PIN)^{commit} 2>/dev/null && \ + git -C $(LV_BINDINGS_DIR) diff --quiet $(LV_BINDINGS_PIN) -- generated/lvgl_circuitpython.c generated/lvgl_circuitpython.h lvgl lv_conf.h && \ + git -C $(LV_BINDINGS_DIR) diff --quiet -- generated/lvgl_circuitpython.c generated/lvgl_circuitpython.h lvgl lv_conf.h || echo 1) + +ifeq ($(LV_BINDINGS_PIN),) +$(error Missing $(LVCP_DIR)/LVGL_BINDINGS_COMMIT) +endif +ifneq ($(LV_BINDINGS_DIRTY),) +$(error $(LV_BINDINGS_DIR) does not match pinned binding inputs $(LV_BINDINGS_PIN); check out that commit or run scripts/sync_from_lvgl_bindings.sh with an exact ref) +endif +$(if $(wildcard $(LVCP_C)),,$(error $(LVCP_C) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython)) +$(if $(wildcard $(LVCP_H)),,$(error $(LVCP_H) not found. Run $(LV_BINDINGS_DIR)/regenerate_all.sh --target circuitpython)) + +# LVGL itself. Its host backends (SDL, X11, Wayland, OpenGL ES, ...) need host +# libraries and are left out of board builds; on unix each compiles to nothing +# unless lv_conf.h turns it on. +LVCP_LVGL_SOURCES := $(shell find $(LVGL_DIR)/src -type f -name '*.c') +ifeq ($(findstring /ports/unix,$(abspath $(CURDIR))),) +LVCP_EXCLUDE_DIRS := \ + $(LVGL_DIR)/src/drivers/opengles \ + $(LVGL_DIR)/src/drivers/sdl \ + $(LVGL_DIR)/src/drivers/glfw \ + $(LVGL_DIR)/src/drivers/x11 \ + $(LVGL_DIR)/src/drivers/wayland \ + $(LVGL_DIR)/src/drivers/evdev \ + $(LVGL_DIR)/src/drivers/libinput \ + $(LVGL_DIR)/src/drivers/qnx \ + $(LVGL_DIR)/src/drivers/uefi \ + $(LVGL_DIR)/src/drivers/nuttx \ + $(LVGL_DIR)/src/drivers/windows \ + $(LVGL_DIR)/src/draw/opengles \ + $(LVGL_DIR)/src/libs/gltf +LVCP_LVGL_SOURCES := $(foreach s,$(LVCP_LVGL_SOURCES),$(if $(strip $(foreach d,$(LVCP_EXCLUDE_DIRS),$(findstring $(d)/,$(s)))),,$(s))) +endif + +# LVGL has no MP_QSTR_* in it, so it goes in SRC_USERMOD_LIB_C, which py.mk +# compiles and links but never scans for qstrs. Scanning it would only cost +# time, and on espressif the scan's command line passes the kernel's limit. +SRC_USERMOD_LIB_C += $(LVCP_LVGL_SOURCES) + +# The CircuitPython side: the module (registered with MP_REGISTER_MODULE in +# shared-bindings/lvgl/__init__.c), the generated bindings, LVGL's allocator on +# CircuitPython's heap, LVGL's JPEG decoder on CircuitPython's TJpgDec (a no-op +# without jpegio), and what an LVGL assertion does (safe mode, not a hang). +# These are scanned for qstrs. +LVCP_SPIKE := $(LVCP_DIR)/src/circuitpython_spike +LVCP_SOURCES := \ + $(LVCP_SPIKE)/shared-bindings/lvgl/__init__.c \ + $(LVCP_SPIKE)/shared-module/lvgl/__init__.c \ + $(LVCP_DIR)/src/lv_mem_core_circuitpython.c \ + $(LVCP_DIR)/src/lv_jpegio_decoder_circuitpython.c \ + $(LVCP_DIR)/src/lv_assert_circuitpython.c \ + $(LVCP_C) +SRC_USERMOD_C += $(LVCP_SOURCES) + +# The spike includes "shared-bindings/lvgl/__init__.h"; CircuitPython's own +# -I$(TOP) comes first and has no lvgl/, so these resolve to the spike's. +# LV_CIRCUITPYTHON_BUILD picks the allocator above in lv_conf.h and leaves +# lvgl.init()/deinit() to the spike. +CFLAGS_USERMOD += -DLV_CIRCUITPYTHON_BUILD=1 -I$(LVCP_SPIKE) -I$(LV_BINDINGS_DIR) -I$(LVGL_DIR) -Wno-unused-function + +# LVGL's bindings bring about 53 KB of qstrs. CircuitPython 11 keeps a ROM qstr +# pool's strings in one blob addressed by 16-bit offsets, 64 KB at most, which +# a board's own modules and LVGL together pass. Where CircuitPython has the +# MICROPY_QSTR_OFFSET_BYTES setting, four-byte offsets lift that limit for +# about two bytes of flash per qstr. +CFLAGS_USERMOD += -DMICROPY_QSTR_OFFSET_BYTES=4 + +# Upstream LVGL and the generated bindings trip warnings CircuitPython's ports +# make errors of, and the board ports append those -Werror flags after +# CFLAGS_USERMOD, so a module-wide -Wno-* would lose. A target-specific flag +# lands last, so they are turned off per object, for this module's objects +# only. -Wfloat-equal, -Wdouble-promotion and -Wcast-align are the board +# ports' (arc and chart compare floats directly); the rest are LVGL's style. +LVCP_OBJ_CFLAGS := -Wno-cast-align -Wno-nested-externs -Wno-unused-parameter \ + -Wno-sign-compare -Wno-missing-prototypes -Wno-old-style-definition \ + -Wno-float-conversion -Wno-double-promotion -Wno-shadow -Wno-type-limits \ + -Wno-suggest-attribute=format -Wno-float-equal -Wno-unused-const-variable \ + -Wno-unused-but-set-variable -Wno-unused-function -Wno-undef + +# py.mk names a user module's object $(BUILD)//.o, and a source outside the module (lvgl-bindings beside it) +# $(BUILD)/.o. Both spellings are declared. +LVCP_OBJ = $(BUILD)/$(patsubst $(LVCP_DIR)/%,$(notdir $(LVCP_DIR))/%,$(1)) $(BUILD)/$(1) +$(foreach _s,$(LVCP_LVGL_SOURCES) $(LVCP_SOURCES),\ + $(eval $(call LVCP_OBJ,$(_s:.c=.o)): CFLAGS += $(LVCP_OBJ_CFLAGS))) diff --git a/scripts/fetch_bindings.sh b/scripts/fetch_bindings.sh new file mode 100755 index 0000000..a18fbf4 --- /dev/null +++ b/scripts/fetch_bindings.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Fetch lvgl-bindings at the commit in LVGL_BINDINGS_COMMIT into +# .deps/lvgl-bindings, with only what this module compiles: generated/, +# lv_conf.h, and LVGL's src/ plus its top-level headers. Shallow, sparse and +# blobless, so it downloads tens of MB rather than LVGL's whole history. +# +# micropython.mk runs this itself when there is no LV_BINDINGS_DIR and no +# lvgl-bindings checkout beside this one (a clone on its own); with a +# sibling checkout the sibling wins and this never runs. +set -euo pipefail +HERE=$(cd "$(dirname "$0")/.." && pwd) +DEST="$HERE/.deps/lvgl-bindings" +URL="${LV_BINDINGS_REPO:-https://github.com/PyDevices/lvgl-bindings.git}" +PIN=$(tr -d '[:space:]' < "$HERE/LVGL_BINDINGS_COMMIT") + +if [[ -d "$DEST/.git" && "$(git -C "$DEST" rev-parse HEAD 2>/dev/null)" == "$PIN" && -f "$DEST/lvgl/lvgl.h" ]]; then + echo "lvgl-bindings ${PIN:0:12} already in .deps" + exit 0 +fi +rm -rf "$DEST" +mkdir -p "$DEST" +git -C "$DEST" init -q +git -C "$DEST" remote add origin "$URL" +git -C "$DEST" config extensions.partialClone origin +git -C "$DEST" sparse-checkout set generated lvgl +git -C "$DEST" fetch -q --depth 1 --filter=blob:none origin "$PIN" +git -C "$DEST" checkout -q FETCH_HEAD +# LVGL itself, as its own partial clone at the commit lvgl-bindings records: +# `git submodule update` ignores --filter and pulls a ~100 MB pack. +LVGL_SHA=$(git -C "$DEST" ls-tree HEAD lvgl | awk '{print $3}') +LVGL_URL=$(git -C "$DEST" config -f .gitmodules submodule.lvgl.url) +rm -rf "$DEST/lvgl" +mkdir -p "$DEST/lvgl" +git -C "$DEST/lvgl" init -q +git -C "$DEST/lvgl" remote add origin "$LVGL_URL" +git -C "$DEST/lvgl" config extensions.partialClone origin +git -C "$DEST/lvgl" sparse-checkout set src +git -C "$DEST/lvgl" fetch -q --depth 1 --filter=blob:none origin "$LVGL_SHA" +git -C "$DEST/lvgl" checkout -q FETCH_HEAD +echo "lvgl-bindings ${PIN:0:12} fetched into $DEST" diff --git a/src/circuitpython_spike/shared-bindings/lvgl/__init__.c b/src/circuitpython_spike/shared-bindings/lvgl/__init__.c index bffcb38..d5ebf10 100644 --- a/src/circuitpython_spike/shared-bindings/lvgl/__init__.c +++ b/src/circuitpython_spike/shared-bindings/lvgl/__init__.c @@ -24,6 +24,38 @@ MP_DEFINE_CONST_FUN_OBJ_0(lvgl_deinit_obj, lvgl_module_deinit_fn); MP_REGISTER_MODULE(MP_QSTR_lvgl, lvgl_module); +#if MICROPY_MODULE_BUILTIN_INIT && MICROPY_MODULE_ATTR_DELEGATION +// CircuitPython starts a new VM on every reload (a saved file, Ctrl-D, +// supervisor.reload()) and does not clear root pointers between VMs. LVGL's +// state, lv_global_t and everything it reaches, lives on the GC heap behind +// them, so in the next VM they point into a heap that has been wiped: +// lv.is_initialized() reads garbage and the first display_create asks for +// megabytes. A built-in module's __init__ runs whenever the import system +// finds it among the built-ins; the generated module dict has none, so this +// delegation answers the lookup and drops the stale state. Built-ins are not +// kept in sys.modules, so __init__ would otherwise run on every +// `import lvgl` and drop live state too: it puts lvgl in sys.modules, which a +// later import in the same VM finds first, and which a new VM starts without. +// (Deleting it from sys.modules, or importing with an empty sys.path, drops +// live state; LVGL is then initialised afresh.) +static mp_obj_t lvgl_module_vm_init(void) { + mp_lv_deinit_gc(); + mp_obj_dict_store(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_loaded_modules_dict)), + MP_OBJ_NEW_QSTR(MP_QSTR_lvgl), MP_OBJ_FROM_PTR(&lvgl_module)); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_0(lvgl_module_vm_init_obj, lvgl_module_vm_init); + +void lvgl_module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); +void lvgl_module_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + (void)self_in; + if (dest[0] == MP_OBJ_NULL && attr == MP_QSTR___init__) { + dest[0] = MP_OBJ_FROM_PTR(&lvgl_module_vm_init_obj); + } +} +MP_REGISTER_MODULE_DELEGATION(lvgl_module, lvgl_module_attr); +#endif + void lvgl_init(void) { shared_modules_lvgl_init(); } diff --git a/src/lv_assert_circuitpython.c b/src/lv_assert_circuitpython.c new file mode 100644 index 0000000..9c1b9bf --- /dev/null +++ b/src/lv_assert_circuitpython.c @@ -0,0 +1,29 @@ +/** + * @file lv_assert_circuitpython.c + * + * What an LVGL assertion does on CircuitPython (lvgl-bindings' lv_conf.h + * names this function as LV_ASSERT_HANDLER when LV_CIRCUITPYTHON_BUILD is + * set). LVGL's default is to spin forever, and a board spinning in C stops + * answering USB: its drive and serial port vanish until someone presses reset. + * Most LVGL assertions on a board are an allocation that failed because the + * heap is full, so instead the board reboots into safe mode, which reports + * "Third-party firmware fatal error." and keeps CIRCUITPY reachable to fix + * code.py. On the unix port the process aborts. + */ +#include "py/mpconfig.h" + +void lv_circuitpython_assert_failed(void); + +#if defined(__unix__) || defined(__APPLE__) +#include + +void lv_circuitpython_assert_failed(void) { + abort(); +} +#else +#include "supervisor/shared/safe_mode.h" + +void lv_circuitpython_assert_failed(void) { + reset_into_safe_mode(SAFE_MODE_SDK_FATAL_ERROR); +} +#endif diff --git a/src/lv_mem_core_circuitpython.c b/src/lv_mem_core_circuitpython.c index e5c8b66..9d4c0a5 100644 --- a/src/lv_mem_core_circuitpython.c +++ b/src/lv_mem_core_circuitpython.c @@ -70,7 +70,12 @@ void *lv_malloc_core(size_t size) * dereferencing a bogus mp_obj type pointer and crashing (SIGSEGV). */ return gc_alloc(size, false); #else - return m_malloc(size); + /* The _maybe allocators return NULL when the heap is full; m_malloc would + * raise MemoryError, a longjmp straight out of LVGL's C that leaves a + * refresh half done (its flushing and rendering flags set) for the next + * pass to spin on. With NULL, LVGL fails the way it was written to: + * LV_ASSERT_MALLOC, which lv_assert_circuitpython.c turns into safe mode. */ + return m_malloc_maybe(size); #endif } @@ -79,7 +84,7 @@ void *lv_realloc_core(void *p, size_t new_size) #if MICROPY_MALLOC_USES_ALLOCATED_SIZE return gc_realloc(p, new_size, true); #else - return m_realloc(p, new_size); + return m_realloc_maybe(p, new_size, true); #endif } diff --git a/tests/test_integration.py b/tests/test_integration.py index aee328b..abaf03d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -62,3 +62,39 @@ def test_jpegio_decoder_shim_is_wired(): assert "file_has_soi" in text assert "lv_fs_get_ext" not in text assert "has_jpeg_ext" not in text + + +def test_user_c_module_keeps_lvgl_out_of_the_qstr_scan(): + make = (ROOT / "micropython.mk").read_text() + # LVGL has no MP_QSTR_*: compiled and linked, never scanned. + assert "SRC_USERMOD_LIB_C += $(LVCP_LVGL_SOURCES)" in make + assert "SRC_USERMOD_C += $(LVCP_SOURCES)" in make + assert "generated/lvgl_circuitpython.c" in make + assert "LVGL_BINDINGS_COMMIT" in make + # gifio's AnimatedGIF collides with LVGL's at link time: stop early. + assert "ifeq ($(CIRCUITPY_GIFIO),1)" in make + # The board ports' late -Werror flags lose to per-object flags only. + assert "CFLAGS += $(LVCP_OBJ_CFLAGS)" in make + + +def test_a_reload_drops_lvgl_state_from_the_previous_vm(): + shared_bindings = ( + ROOT / "src/circuitpython_spike/shared-bindings/lvgl/__init__.c" + ).read_text() + assert "MP_REGISTER_MODULE_DELEGATION(lvgl_module, lvgl_module_attr);" in shared_bindings + assert "attr == MP_QSTR___init__" in shared_bindings + assert "mp_lv_deinit_gc();" in shared_bindings + # Built-ins stay out of sys.modules, so without this every `import lvgl` + # would run __init__ again and drop live state. + assert "MP_STATE_VM(mp_loaded_modules_dict)" in shared_bindings + + +def test_an_lvgl_failure_never_spins_a_board_off_usb(): + mem = (ROOT / "src/lv_mem_core_circuitpython.c").read_text() + # NULL, not MemoryError: an exception would longjmp out of LVGL's C. + assert "m_malloc_maybe(size)" in mem + assert "m_realloc_maybe(p, new_size, true)" in mem + handler = (ROOT / "src/lv_assert_circuitpython.c").read_text() + assert "reset_into_safe_mode(SAFE_MODE_SDK_FATAL_ERROR)" in handler + assert "src/lv_assert_circuitpython.c" in (ROOT / "micropython.mk").read_text() + assert "src/lv_assert_circuitpython.c" in (ROOT / "circuitpython.mk").read_text()