From ee1ee7ce690a8e9c6b1591aea457eacd462a34a6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:30:57 +0900 Subject: [PATCH] CI: probe embedding the 3.15t wasm build without main (#175) Exports wasmExports, wasmMemory and wasmTable from the free-threaded wasm build and adds a node probe step to build-ft-wasm.yml: the shared memory, a non-shared versus shared memory import (what Kotlin/Wasm emits versus what a patch would emit), and Py_Initialize from the JS main thread with four threads and gc.collect() in a worker. The step runs after the archive upload, so a probe failure does not lose the archive. --- .github/scripts/ft-wasm/build-wasm.sh | 54 ++++++++ .github/scripts/ft-wasm/embed-probe.mjs | 167 ++++++++++++++++++++++++ .github/workflows/build-ft-wasm.yml | 21 +++ docs/platforms/free-threaded-wasm.md | 38 ++++++ 4 files changed, 280 insertions(+) create mode 100644 .github/scripts/ft-wasm/embed-probe.mjs diff --git a/.github/scripts/ft-wasm/build-wasm.sh b/.github/scripts/ft-wasm/build-wasm.sh index 32886130..5e9e86cf 100755 --- a/.github/scripts/ft-wasm/build-wasm.sh +++ b/.github/scripts/ft-wasm/build-wasm.sh @@ -150,6 +150,59 @@ grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \ grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t" log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t" +# ---- patch: JS reachability for the embedding probe ----------------------- +# Same substitution as patch_makefile_reachability in python-multiplatform/scripts/wasm/build-cpython.sh: +# the library reaches CPython without main through Module.wasmExports / Module.wasmMemory, which +# the stock link line does not expose (3.15 default: FS,callMain,ENV,HEAPU32,TTY). Written as a +# regex over the whole comma list rather than a fixed string, because the 3.15 Makefile was not +# available when this was written. The result is asserted, so a changed link line fails the +# build here instead of producing an archive the probe cannot use. +# The C functions: CPython's LINKFORSHARED lists -sEXPORTED_FUNCTIONS explicitly +# (3.14: _main,_Py_Version,__PyRuntime,...) next to -sMAIN_MODULE. Whether 3.15 still does is +# unverified, so _malloc,_free,_Py_Version are appended to that list when it exists (harmless if +# MAIN_MODULE already exports everything); the probe then checks the exports it needs and +# fails loudly if one is missing. +patch_makefile_exports() { + python3 - "$hostdir/Makefile" <<'PATCH' +import re +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() + +def add_runtime_methods(m): + items = m.group(2).split(",") + for extra in ("wasmExports", "wasmMemory", "wasmTable"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, n = re.subn(r"(-sEXPORTED_RUNTIME_METHODS=)(\S+)", add_runtime_methods, t) +assert n >= 1, "no -sEXPORTED_RUNTIME_METHODS= in the host Makefile; the link line changed shape, re-check the patch" +for m in re.finditer(r"-sEXPORTED_RUNTIME_METHODS=(\S+)", t): + methods = m.group(1).split(",") + assert all(x in methods for x in ("wasmExports", "wasmMemory", "wasmTable")), "wasmExports/wasmMemory/wasmTable missing after the patch: " + m.group(0) +print(f"Makefile: wasmExports,wasmMemory,wasmTable added to {n} -sEXPORTED_RUNTIME_METHODS list(s)") + +def add_functions(m): + items = m.group(2).split(",") + for extra in ("_malloc", "_free", "_Py_Version"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, k = re.subn(r"(-sEXPORTED_FUNCTIONS=)(\S+)", add_functions, t) +if k: + print(f"Makefile: _malloc,_free,_Py_Version ensured in {k} -sEXPORTED_FUNCTIONS list(s)") +else: + print("Makefile: NOTE no -sEXPORTED_FUNCTIONS= found; relying on the link exporting C symbols another way (the probe checks)") +p.write_text(t) +for line in t.splitlines(): + if "-sEXPORTED_RUNTIME_METHODS" in line or "-sEXPORTED_FUNCTIONS" in line or "-sMAIN_MODULE" in line: + print("Makefile link line:", line.strip()[:2000]) +PATCH +} +patch_makefile_exports + # ---- host build ----------------------------------------------------------- # make-host also runs `python.sh --version`, so a startup failure (for example the trampoline # instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step. @@ -222,6 +275,7 @@ cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/" echo "CFLAGS=$HOST_CFLAGS" echo "LDFLAGS=$HOST_LDFLAGS" echo "trampoline patch: $TRAMPOLINE_PATCH" + echo "link patch: wasmExports,wasmMemory,wasmTable in EXPORTED_RUNTIME_METHODS (embedding probe)" echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)" } > "$stage/BUILD-INFO.txt" diff --git a/.github/scripts/ft-wasm/embed-probe.mjs b/.github/scripts/ft-wasm/embed-probe.mjs new file mode 100644 index 00000000..375d2239 --- /dev/null +++ b/.github/scripts/ft-wasm/embed-probe.mjs @@ -0,0 +1,167 @@ +// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal). +// +// Usage: node embed-probe.mjs +// +// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory +// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the +// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final +// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller +// wraps this in `timeout` and reads the last "[x]" line it printed. +// +// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and +// services proxied calls, so this probe says nothing about the browser. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const root = path.resolve(process.argv[2] ?? ""); +if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) { + console.log(`[setup] FAIL: usage: node embed-probe.mjs , got '${process.argv[2]}'`); + process.exit(2); +} + +let failed = false; +const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`); +const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); }; +const finish = () => { + if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); } + console.log("FT-WASM-EMBED-PROBE-OK"); + process.exit(0); // the pthread Workers would otherwise keep the process alive +}; + +// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a +// JSPI-suspending syscall import would throw SuspendError. +delete WebAssembly.promising; +delete WebAssembly.Suspending; + +const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d)); +if (!libDir) { fail("a", `no lib/python3.t in ${root}`); finish(); } +const minor = libDir.replace(/^python/, ""); + +// ---- a. load the factory without main ----------------------------------------------------------- +let M; +try { + const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href); + M = await EmscriptenModule({ + noInitialRun: true, + thisProgram: `${root}/bin/python${minor}`, + arguments: [], + preRun(Module) { + globalThis.Module = Module; + // Minimum of node_entry.mjs: mount the host filesystem (the archive is read through + // NODEFS) except the three directories Emscripten owns, then set the environment. + for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } + Module.FS.chdir(root); + Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; + Module.ENV.PYTHONHOME = root; + }, + }); + const mem = M.wasmMemory; + if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer"); + pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`); +} catch (e) { + fail("a", String(e && e.stack || e)); + finish(); +} +const E = M.wasmExports; +// A fresh view each time: a shared memory's old buffer keeps its old length after growth. +const u8 = () => new Uint8Array(M.wasmMemory.buffer); +const u32 = () => new Uint32Array(M.wasmMemory.buffer); +const exported = (name) => { + let v = E[name]; + if (v === undefined) v = M["_" + name]; + if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`); + return v; +}; +const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; }; + +// ---- b. memory import checks A1 / A2 ------------------------------------------------------------ +// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and +// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2 +// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit). +const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; +const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f]; +const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02]; +const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00, + 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b]; +const mod = (limits) => new Uint8Array([...head, ...typeSec, + 0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]); +const A1 = mod([0x00, 0x00]); +const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]); +try { + let a1; + try { + new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } }); + a1 = "instantiated"; + } catch (e) { + a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`; + if (a1 === "LinkError") console.log(` A1 message: ${e.message}`); + } + if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`); + const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } }); + const addr = addrOf("Py_Version"); + const viaLoad = a2.exports.load(addr) >>> 0; + const viaHeap = u32()[addr >>> 2]; + if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`); + pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`); +} catch (e) { + fail("b", String(e && e.stack || e)); +} + +// ---- c. embedding without main, threads, gc in a worker ----------------------------------------- +const cstr = (s) => { + const bytes = Buffer.from(s + "\0"); + const p = E.malloc(bytes.length); + if (!p) throw new Error("malloc returned 0"); + u8().set(bytes, p); + return p; +}; +const script = ` +import gc, os, sys, threading +assert sys._is_gil_enabled() is False, "GIL is enabled at the start" +sizes = {} +def reader(i): + with open(os.__file__) as f: + n = len(f.read()) + print(f"embed-probe thread {i} read {n} chars of os.py", flush=True) + sizes[i] = n +threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running: {alive}" +assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes +assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads" +collected = [] +gc_thread = threading.Thread(target=lambda: collected.append(gc.collect())) +gc_thread.start() +gc_thread.join(timeout=120) +assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish" +print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True) +assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc" +`; +try { + E.Py_Initialize(); + pass("c1", "Py_Initialize returned"); + const tstate = E.PyEval_SaveThread(); + pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`); + const gil = E.PyGILState_Ensure(); + const code = cstr(script); + console.log(" running the thread script (a hang here is caught by the caller's timeout)"); + const rc = E.PyRun_SimpleString(code); + E.free(code); + E.PyGILState_Release(gil); + if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`); + pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned"); +} catch (e) { + fail("c", String(e && e.stack || e)); +} +finish(); diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml index b0242b7e..6baebdbe 100644 --- a/.github/workflows/build-ft-wasm.yml +++ b/.github/workflows/build-ft-wasm.yml @@ -83,3 +83,24 @@ jobs: name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten path: ${{ env.FT_OUT }}/*.tar.gz if-no-files-found: error + + # Step 2 of #175 (agent proposal): embed CPython without main from the node main thread, as + # the library does. A separate step after the upload, so a probe failure or hang does not + # lose the archive. Node only: node uses a real Atomics.wait on the main thread, a browser + # busy-waits, so this says nothing about the browser. + - name: Embedding probe (no main) + run: | + root="$(ls -d "$FT_WORK"/wasm-check/python-*t-wasm32-emscripten | head -n 1)" + [ -d "$root" ] || { echo "no unpacked archive under $FT_WORK/wasm-check"; exit 1; } + log="$FT_WORK/embed-probe.log" + set +e + timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs "$root" > "$log" 2>&1 + rc=$? + set -e + cat "$log" + if [ "$rc" -ne 0 ]; then + last="$(grep -E '^\[[a-z0-9]+\]' "$log" | tail -n 1)" + echo "::error::embedding probe exited $rc (124 = timed out, a hang); last check printed: ${last:-none}" + exit "$rc" + fi + grep -q '^FT-WASM-EMBED-PROBE-OK$' "$log" diff --git a/docs/platforms/free-threaded-wasm.md b/docs/platforms/free-threaded-wasm.md index aab98375..c3fe705d 100644 --- a/docs/platforms/free-threaded-wasm.md +++ b/docs/platforms/free-threaded-wasm.md @@ -117,6 +117,44 @@ The script stops at the first failed check, in this order: The archive is uploaded as the workflow artifact `python-t-wasm32-emscripten`. No release is published or changed. +## Embedding probe (step 2 of #175) + +Agent proposal toward #175 step 2, not a decision. It answers, on CI, whether the library's way +of using CPython (load `python.mjs` as a factory with `noInitialRun`, never call `main`, call +exports from the JS main thread) works against this pthreads build. + +`build-wasm.sh` now appends `wasmExports,wasmMemory,wasmTable` to every +`-sEXPORTED_RUNTIME_METHODS` list in the host Makefile before `make-host` (the same edit as +`patch_makefile_reachability` in `python-multiplatform/scripts/wasm/build-cpython.sh`, as a +regex because the 3.15 Makefile text was not checked) and asserts the result. If the Makefile +has a `-sEXPORTED_FUNCTIONS=` list it also ensures `_malloc,_free,_Py_Version` are in it. A +missing list is only logged: whether 3.15 still exports C symbols through `-sMAIN_MODULE` +(as 3.14 does) is unverified, and the probe fails loudly if an export it needs is absent. + +The workflow step `Embedding probe (no main)` runs after the archive upload, so a probe +failure or hang does not lose the archive: +`timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs `. It prints one +`[x] PASS|FAIL` line per check and `FT-WASM-EMBED-PROBE-OK` only if all pass; on a timeout +(exit 124) the step reports the last check line printed. + +- `a`: the factory loads with `noInitialRun`, `Module.wasmMemory.buffer` is a + `SharedArrayBuffer` (page count printed). +- `b`: two hand-assembled modules import `intrinsics.memory` (the name Kotlin/Wasm uses). + A1 imports a non-shared memory and must fail with `LinkError`; A2 imports a shared one with + maximum 65536 pages and must instantiate, and `A2.load(&Py_Version)` must equal the value + read through a view of `wasmMemory`. A2 passing keeps the post-link import patch (investigation + options 2a/2b) open; A1 not failing would mean the blocker is not real. +- `c`: `Py_Initialize`, `PyEval_SaveThread`, then `PyGILState_Ensure` and `PyRun_SimpleString` + (strings placed with the exported `malloc`) of a script that starts four threads which print + and read `os.py`, joins them, asserts `sys._is_gil_enabled()` is False, and runs + `gc.collect()` inside a worker thread. This exercises the main thread blocking in `join` + while Workers need it to service proxied syscalls. + +Node only. Node's main thread uses a real `Atomics.wait`; a browser's main thread busy-waits +and services proxied calls, so a pass here does not show the browser works, and a Node deadlock +may not occur in a browser. It does not involve Kotlin at all (no upcalls, no proxies, no +Kotlin-side memory import). + ## Not verified - The workflow has not run yet. Whether 3.15.0 configures, compiles and links with