Skip to content

Commit 2bca2c8

Browse files
authored
Merge pull request #411 from thisisthepy/feat/wasm-ft-embed-probe
CI: probe embedding the 3.15t wasm build without main (#175 step 2)
2 parents 305378e + ee1ee7c commit 2bca2c8

4 files changed

Lines changed: 280 additions & 0 deletions

File tree

‎.github/scripts/ft-wasm/build-wasm.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,59 @@ grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \
150150
grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t"
151151
log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t"
152152

153+
# ---- patch: JS reachability for the embedding probe -----------------------
154+
# Same substitution as patch_makefile_reachability in python-multiplatform/scripts/wasm/build-cpython.sh:
155+
# the library reaches CPython without main through Module.wasmExports / Module.wasmMemory, which
156+
# the stock link line does not expose (3.15 default: FS,callMain,ENV,HEAPU32,TTY). Written as a
157+
# regex over the whole comma list rather than a fixed string, because the 3.15 Makefile was not
158+
# available when this was written. The result is asserted, so a changed link line fails the
159+
# build here instead of producing an archive the probe cannot use.
160+
# The C functions: CPython's LINKFORSHARED lists -sEXPORTED_FUNCTIONS explicitly
161+
# (3.14: _main,_Py_Version,__PyRuntime,...) next to -sMAIN_MODULE. Whether 3.15 still does is
162+
# unverified, so _malloc,_free,_Py_Version are appended to that list when it exists (harmless if
163+
# MAIN_MODULE already exports everything); the probe then checks the exports it needs and
164+
# fails loudly if one is missing.
165+
patch_makefile_exports() {
166+
python3 - "$hostdir/Makefile" <<'PATCH'
167+
import re
168+
import sys
169+
from pathlib import Path
170+
p = Path(sys.argv[1]); t = p.read_text()
171+
172+
def add_runtime_methods(m):
173+
items = m.group(2).split(",")
174+
for extra in ("wasmExports", "wasmMemory", "wasmTable"):
175+
if extra not in items:
176+
items.append(extra)
177+
return m.group(1) + ",".join(items)
178+
179+
t, n = re.subn(r"(-sEXPORTED_RUNTIME_METHODS=)(\S+)", add_runtime_methods, t)
180+
assert n >= 1, "no -sEXPORTED_RUNTIME_METHODS= in the host Makefile; the link line changed shape, re-check the patch"
181+
for m in re.finditer(r"-sEXPORTED_RUNTIME_METHODS=(\S+)", t):
182+
methods = m.group(1).split(",")
183+
assert all(x in methods for x in ("wasmExports", "wasmMemory", "wasmTable")), "wasmExports/wasmMemory/wasmTable missing after the patch: " + m.group(0)
184+
print(f"Makefile: wasmExports,wasmMemory,wasmTable added to {n} -sEXPORTED_RUNTIME_METHODS list(s)")
185+
186+
def add_functions(m):
187+
items = m.group(2).split(",")
188+
for extra in ("_malloc", "_free", "_Py_Version"):
189+
if extra not in items:
190+
items.append(extra)
191+
return m.group(1) + ",".join(items)
192+
193+
t, k = re.subn(r"(-sEXPORTED_FUNCTIONS=)(\S+)", add_functions, t)
194+
if k:
195+
print(f"Makefile: _malloc,_free,_Py_Version ensured in {k} -sEXPORTED_FUNCTIONS list(s)")
196+
else:
197+
print("Makefile: NOTE no -sEXPORTED_FUNCTIONS= found; relying on the link exporting C symbols another way (the probe checks)")
198+
p.write_text(t)
199+
for line in t.splitlines():
200+
if "-sEXPORTED_RUNTIME_METHODS" in line or "-sEXPORTED_FUNCTIONS" in line or "-sMAIN_MODULE" in line:
201+
print("Makefile link line:", line.strip()[:2000])
202+
PATCH
203+
}
204+
patch_makefile_exports
205+
153206
# ---- host build -----------------------------------------------------------
154207
# make-host also runs `python.sh --version`, so a startup failure (for example the trampoline
155208
# instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step.
@@ -222,6 +275,7 @@ cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/"
222275
echo "CFLAGS=$HOST_CFLAGS"
223276
echo "LDFLAGS=$HOST_LDFLAGS"
224277
echo "trampoline patch: $TRAMPOLINE_PATCH"
278+
echo "link patch: wasmExports,wasmMemory,wasmTable in EXPORTED_RUNTIME_METHODS (embedding probe)"
225279
echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)"
226280
} > "$stage/BUILD-INFO.txt"
227281

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal).
2+
//
3+
// Usage: node embed-probe.mjs <archive root>
4+
//
5+
// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory
6+
// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the
7+
// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final
8+
// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller
9+
// wraps this in `timeout` and reads the last "[x]" line it printed.
10+
//
11+
// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and
12+
// services proxied calls, so this probe says nothing about the browser.
13+
import fs from "node:fs";
14+
import path from "node:path";
15+
import { pathToFileURL } from "node:url";
16+
17+
const root = path.resolve(process.argv[2] ?? "");
18+
if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) {
19+
console.log(`[setup] FAIL: usage: node embed-probe.mjs <archive root containing python.mjs>, got '${process.argv[2]}'`);
20+
process.exit(2);
21+
}
22+
23+
let failed = false;
24+
const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`);
25+
const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); };
26+
const finish = () => {
27+
if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); }
28+
console.log("FT-WASM-EMBED-PROBE-OK");
29+
process.exit(0); // the pthread Workers would otherwise keep the process alive
30+
};
31+
32+
// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a
33+
// JSPI-suspending syscall import would throw SuspendError.
34+
delete WebAssembly.promising;
35+
delete WebAssembly.Suspending;
36+
37+
const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d));
38+
if (!libDir) { fail("a", `no lib/python3.<minor>t in ${root}`); finish(); }
39+
const minor = libDir.replace(/^python/, "");
40+
41+
// ---- a. load the factory without main -----------------------------------------------------------
42+
let M;
43+
try {
44+
const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href);
45+
M = await EmscriptenModule({
46+
noInitialRun: true,
47+
thisProgram: `${root}/bin/python${minor}`,
48+
arguments: [],
49+
preRun(Module) {
50+
globalThis.Module = Module;
51+
// Minimum of node_entry.mjs: mount the host filesystem (the archive is read through
52+
// NODEFS) except the three directories Emscripten owns, then set the environment.
53+
for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) {
54+
Module.FS.mkdirTree(dir);
55+
Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir);
56+
}
57+
Module.FS.chdir(root);
58+
Object.assign(Module.ENV, process.env);
59+
delete Module.ENV.PATH;
60+
Module.ENV.PYTHONHOME = root;
61+
},
62+
});
63+
const mem = M.wasmMemory;
64+
if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)");
65+
if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)");
66+
if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer");
67+
pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`);
68+
} catch (e) {
69+
fail("a", String(e && e.stack || e));
70+
finish();
71+
}
72+
const E = M.wasmExports;
73+
// A fresh view each time: a shared memory's old buffer keeps its old length after growth.
74+
const u8 = () => new Uint8Array(M.wasmMemory.buffer);
75+
const u32 = () => new Uint32Array(M.wasmMemory.buffer);
76+
const exported = (name) => {
77+
let v = E[name];
78+
if (v === undefined) v = M["_" + name];
79+
if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`);
80+
return v;
81+
};
82+
const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; };
83+
84+
// ---- b. memory import checks A1 / A2 ------------------------------------------------------------
85+
// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and
86+
// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2
87+
// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit).
88+
const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00];
89+
const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f];
90+
const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02];
91+
const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00,
92+
0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b];
93+
const mod = (limits) => new Uint8Array([...head, ...typeSec,
94+
0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]);
95+
const A1 = mod([0x00, 0x00]);
96+
const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]);
97+
try {
98+
let a1;
99+
try {
100+
new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } });
101+
a1 = "instantiated";
102+
} catch (e) {
103+
a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`;
104+
if (a1 === "LinkError") console.log(` A1 message: ${e.message}`);
105+
}
106+
if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`);
107+
const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } });
108+
const addr = addrOf("Py_Version");
109+
const viaLoad = a2.exports.load(addr) >>> 0;
110+
const viaHeap = u32()[addr >>> 2];
111+
if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`);
112+
pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`);
113+
} catch (e) {
114+
fail("b", String(e && e.stack || e));
115+
}
116+
117+
// ---- c. embedding without main, threads, gc in a worker -----------------------------------------
118+
const cstr = (s) => {
119+
const bytes = Buffer.from(s + "\0");
120+
const p = E.malloc(bytes.length);
121+
if (!p) throw new Error("malloc returned 0");
122+
u8().set(bytes, p);
123+
return p;
124+
};
125+
const script = `
126+
import gc, os, sys, threading
127+
assert sys._is_gil_enabled() is False, "GIL is enabled at the start"
128+
sizes = {}
129+
def reader(i):
130+
with open(os.__file__) as f:
131+
n = len(f.read())
132+
print(f"embed-probe thread {i} read {n} chars of os.py", flush=True)
133+
sizes[i] = n
134+
threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)]
135+
for t in threads:
136+
t.start()
137+
for t in threads:
138+
t.join(timeout=120)
139+
alive = [t.name for t in threads if t.is_alive()]
140+
assert not alive, f"threads still running: {alive}"
141+
assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes
142+
assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads"
143+
collected = []
144+
gc_thread = threading.Thread(target=lambda: collected.append(gc.collect()))
145+
gc_thread.start()
146+
gc_thread.join(timeout=120)
147+
assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish"
148+
print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True)
149+
assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc"
150+
`;
151+
try {
152+
E.Py_Initialize();
153+
pass("c1", "Py_Initialize returned");
154+
const tstate = E.PyEval_SaveThread();
155+
pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`);
156+
const gil = E.PyGILState_Ensure();
157+
const code = cstr(script);
158+
console.log(" running the thread script (a hang here is caught by the caller's timeout)");
159+
const rc = E.PyRun_SimpleString(code);
160+
E.free(code);
161+
E.PyGILState_Release(gil);
162+
if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`);
163+
pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned");
164+
} catch (e) {
165+
fail("c", String(e && e.stack || e));
166+
}
167+
finish();

‎.github/workflows/build-ft-wasm.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,24 @@ jobs:
8383
name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten
8484
path: ${{ env.FT_OUT }}/*.tar.gz
8585
if-no-files-found: error
86+
87+
# Step 2 of #175 (agent proposal): embed CPython without main from the node main thread, as
88+
# the library does. A separate step after the upload, so a probe failure or hang does not
89+
# lose the archive. Node only: node uses a real Atomics.wait on the main thread, a browser
90+
# busy-waits, so this says nothing about the browser.
91+
- name: Embedding probe (no main)
92+
run: |
93+
root="$(ls -d "$FT_WORK"/wasm-check/python-*t-wasm32-emscripten | head -n 1)"
94+
[ -d "$root" ] || { echo "no unpacked archive under $FT_WORK/wasm-check"; exit 1; }
95+
log="$FT_WORK/embed-probe.log"
96+
set +e
97+
timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs "$root" > "$log" 2>&1
98+
rc=$?
99+
set -e
100+
cat "$log"
101+
if [ "$rc" -ne 0 ]; then
102+
last="$(grep -E '^\[[a-z0-9]+\]' "$log" | tail -n 1)"
103+
echo "::error::embedding probe exited $rc (124 = timed out, a hang); last check printed: ${last:-none}"
104+
exit "$rc"
105+
fi
106+
grep -q '^FT-WASM-EMBED-PROBE-OK$' "$log"

‎docs/platforms/free-threaded-wasm.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,44 @@ The script stops at the first failed check, in this order:
117117
The archive is uploaded as the workflow artifact `python-<version>t-wasm32-emscripten`. No
118118
release is published or changed.
119119

120+
## Embedding probe (step 2 of #175)
121+
122+
Agent proposal toward #175 step 2, not a decision. It answers, on CI, whether the library's way
123+
of using CPython (load `python.mjs` as a factory with `noInitialRun`, never call `main`, call
124+
exports from the JS main thread) works against this pthreads build.
125+
126+
`build-wasm.sh` now appends `wasmExports,wasmMemory,wasmTable` to every
127+
`-sEXPORTED_RUNTIME_METHODS` list in the host Makefile before `make-host` (the same edit as
128+
`patch_makefile_reachability` in `python-multiplatform/scripts/wasm/build-cpython.sh`, as a
129+
regex because the 3.15 Makefile text was not checked) and asserts the result. If the Makefile
130+
has a `-sEXPORTED_FUNCTIONS=` list it also ensures `_malloc,_free,_Py_Version` are in it. A
131+
missing list is only logged: whether 3.15 still exports C symbols through `-sMAIN_MODULE`
132+
(as 3.14 does) is unverified, and the probe fails loudly if an export it needs is absent.
133+
134+
The workflow step `Embedding probe (no main)` runs after the archive upload, so a probe
135+
failure or hang does not lose the archive:
136+
`timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs <unpacked archive>`. It prints one
137+
`[x] PASS|FAIL` line per check and `FT-WASM-EMBED-PROBE-OK` only if all pass; on a timeout
138+
(exit 124) the step reports the last check line printed.
139+
140+
- `a`: the factory loads with `noInitialRun`, `Module.wasmMemory.buffer` is a
141+
`SharedArrayBuffer` (page count printed).
142+
- `b`: two hand-assembled modules import `intrinsics.memory` (the name Kotlin/Wasm uses).
143+
A1 imports a non-shared memory and must fail with `LinkError`; A2 imports a shared one with
144+
maximum 65536 pages and must instantiate, and `A2.load(&Py_Version)` must equal the value
145+
read through a view of `wasmMemory`. A2 passing keeps the post-link import patch (investigation
146+
options 2a/2b) open; A1 not failing would mean the blocker is not real.
147+
- `c`: `Py_Initialize`, `PyEval_SaveThread`, then `PyGILState_Ensure` and `PyRun_SimpleString`
148+
(strings placed with the exported `malloc`) of a script that starts four threads which print
149+
and read `os.py`, joins them, asserts `sys._is_gil_enabled()` is False, and runs
150+
`gc.collect()` inside a worker thread. This exercises the main thread blocking in `join`
151+
while Workers need it to service proxied syscalls.
152+
153+
Node only. Node's main thread uses a real `Atomics.wait`; a browser's main thread busy-waits
154+
and services proxied calls, so a pass here does not show the browser works, and a Node deadlock
155+
may not occur in a browser. It does not involve Kotlin at all (no upcalls, no proxies, no
156+
Kotlin-side memory import).
157+
120158
## Not verified
121159

122160
- The workflow has not run yet. Whether 3.15.0 configures, compiles and links with

0 commit comments

Comments
 (0)