Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .github/scripts/ft-wasm/build-wasm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,59 @@ grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \
grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t"
log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t"

# ---- patch: JS reachability for the embedding probe -----------------------
# Same substitution as patch_makefile_reachability in python-multiplatform/scripts/wasm/build-cpython.sh:
# the library reaches CPython without main through Module.wasmExports / Module.wasmMemory, which
# the stock link line does not expose (3.15 default: FS,callMain,ENV,HEAPU32,TTY). Written as a
# regex over the whole comma list rather than a fixed string, because the 3.15 Makefile was not
# available when this was written. The result is asserted, so a changed link line fails the
# build here instead of producing an archive the probe cannot use.
# The C functions: CPython's LINKFORSHARED lists -sEXPORTED_FUNCTIONS explicitly
# (3.14: _main,_Py_Version,__PyRuntime,...) next to -sMAIN_MODULE. Whether 3.15 still does is
# unverified, so _malloc,_free,_Py_Version are appended to that list when it exists (harmless if
# MAIN_MODULE already exports everything); the probe then checks the exports it needs and
# fails loudly if one is missing.
patch_makefile_exports() {
python3 - "$hostdir/Makefile" <<'PATCH'
import re
import sys
from pathlib import Path
p = Path(sys.argv[1]); t = p.read_text()

def add_runtime_methods(m):
items = m.group(2).split(",")
for extra in ("wasmExports", "wasmMemory", "wasmTable"):
if extra not in items:
items.append(extra)
return m.group(1) + ",".join(items)

t, n = re.subn(r"(-sEXPORTED_RUNTIME_METHODS=)(\S+)", add_runtime_methods, t)
assert n >= 1, "no -sEXPORTED_RUNTIME_METHODS= in the host Makefile; the link line changed shape, re-check the patch"
for m in re.finditer(r"-sEXPORTED_RUNTIME_METHODS=(\S+)", t):
methods = m.group(1).split(",")
assert all(x in methods for x in ("wasmExports", "wasmMemory", "wasmTable")), "wasmExports/wasmMemory/wasmTable missing after the patch: " + m.group(0)
print(f"Makefile: wasmExports,wasmMemory,wasmTable added to {n} -sEXPORTED_RUNTIME_METHODS list(s)")

def add_functions(m):
items = m.group(2).split(",")
for extra in ("_malloc", "_free", "_Py_Version"):
if extra not in items:
items.append(extra)
return m.group(1) + ",".join(items)

t, k = re.subn(r"(-sEXPORTED_FUNCTIONS=)(\S+)", add_functions, t)
if k:
print(f"Makefile: _malloc,_free,_Py_Version ensured in {k} -sEXPORTED_FUNCTIONS list(s)")
else:
print("Makefile: NOTE no -sEXPORTED_FUNCTIONS= found; relying on the link exporting C symbols another way (the probe checks)")
p.write_text(t)
for line in t.splitlines():
if "-sEXPORTED_RUNTIME_METHODS" in line or "-sEXPORTED_FUNCTIONS" in line or "-sMAIN_MODULE" in line:
print("Makefile link line:", line.strip()[:2000])
PATCH
}
patch_makefile_exports

# ---- host build -----------------------------------------------------------
# make-host also runs `python.sh --version`, so a startup failure (for example the trampoline
# instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step.
Expand Down Expand Up @@ -222,6 +275,7 @@ cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/"
echo "CFLAGS=$HOST_CFLAGS"
echo "LDFLAGS=$HOST_LDFLAGS"
echo "trampoline patch: $TRAMPOLINE_PATCH"
echo "link patch: wasmExports,wasmMemory,wasmTable in EXPORTED_RUNTIME_METHODS (embedding probe)"
echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)"
} > "$stage/BUILD-INFO.txt"

Expand Down
167 changes: 167 additions & 0 deletions .github/scripts/ft-wasm/embed-probe.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal).
//
// Usage: node embed-probe.mjs <archive root>
//
// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory
// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the
// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final
// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller
// wraps this in `timeout` and reads the last "[x]" line it printed.
//
// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and
// services proxied calls, so this probe says nothing about the browser.
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";

const root = path.resolve(process.argv[2] ?? "");
if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) {
console.log(`[setup] FAIL: usage: node embed-probe.mjs <archive root containing python.mjs>, got '${process.argv[2]}'`);
process.exit(2);
}

let failed = false;
const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`);
const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); };
const finish = () => {
if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); }
console.log("FT-WASM-EMBED-PROBE-OK");
process.exit(0); // the pthread Workers would otherwise keep the process alive
};

// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a
// JSPI-suspending syscall import would throw SuspendError.
delete WebAssembly.promising;
delete WebAssembly.Suspending;

const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d));
if (!libDir) { fail("a", `no lib/python3.<minor>t in ${root}`); finish(); }
const minor = libDir.replace(/^python/, "");

// ---- a. load the factory without main -----------------------------------------------------------
let M;
try {
const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href);
M = await EmscriptenModule({
noInitialRun: true,
thisProgram: `${root}/bin/python${minor}`,
arguments: [],
preRun(Module) {
globalThis.Module = Module;
// Minimum of node_entry.mjs: mount the host filesystem (the archive is read through
// NODEFS) except the three directories Emscripten owns, then set the environment.
for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) {
Module.FS.mkdirTree(dir);
Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir);
}
Module.FS.chdir(root);
Object.assign(Module.ENV, process.env);
delete Module.ENV.PATH;
Module.ENV.PYTHONHOME = root;
},
});
const mem = M.wasmMemory;
if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)");
if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)");
if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer");
pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`);
} catch (e) {
fail("a", String(e && e.stack || e));
finish();
}
const E = M.wasmExports;
// A fresh view each time: a shared memory's old buffer keeps its old length after growth.
const u8 = () => new Uint8Array(M.wasmMemory.buffer);
const u32 = () => new Uint32Array(M.wasmMemory.buffer);
const exported = (name) => {
let v = E[name];
if (v === undefined) v = M["_" + name];
if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`);
return v;
};
const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; };

// ---- b. memory import checks A1 / A2 ------------------------------------------------------------
// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and
// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2
// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit).
const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00];
const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f];
const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02];
const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00,
0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b];
const mod = (limits) => new Uint8Array([...head, ...typeSec,
0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]);
const A1 = mod([0x00, 0x00]);
const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]);
try {
let a1;
try {
new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } });
a1 = "instantiated";
} catch (e) {
a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`;
if (a1 === "LinkError") console.log(` A1 message: ${e.message}`);
}
if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`);
const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } });
const addr = addrOf("Py_Version");
const viaLoad = a2.exports.load(addr) >>> 0;
const viaHeap = u32()[addr >>> 2];
if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`);
pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`);
} catch (e) {
fail("b", String(e && e.stack || e));
}

// ---- c. embedding without main, threads, gc in a worker -----------------------------------------
const cstr = (s) => {
const bytes = Buffer.from(s + "\0");
const p = E.malloc(bytes.length);
if (!p) throw new Error("malloc returned 0");
u8().set(bytes, p);
return p;
};
const script = `
import gc, os, sys, threading
assert sys._is_gil_enabled() is False, "GIL is enabled at the start"
sizes = {}
def reader(i):
with open(os.__file__) as f:
n = len(f.read())
print(f"embed-probe thread {i} read {n} chars of os.py", flush=True)
sizes[i] = n
threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=120)
alive = [t.name for t in threads if t.is_alive()]
assert not alive, f"threads still running: {alive}"
assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes
assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads"
collected = []
gc_thread = threading.Thread(target=lambda: collected.append(gc.collect()))
gc_thread.start()
gc_thread.join(timeout=120)
assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish"
print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True)
assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc"
`;
try {
E.Py_Initialize();
pass("c1", "Py_Initialize returned");
const tstate = E.PyEval_SaveThread();
pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`);
const gil = E.PyGILState_Ensure();
const code = cstr(script);
console.log(" running the thread script (a hang here is caught by the caller's timeout)");
const rc = E.PyRun_SimpleString(code);
E.free(code);
E.PyGILState_Release(gil);
if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`);
pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned");
} catch (e) {
fail("c", String(e && e.stack || e));
}
finish();
21 changes: 21 additions & 0 deletions .github/workflows/build-ft-wasm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,24 @@ jobs:
name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten
path: ${{ env.FT_OUT }}/*.tar.gz
if-no-files-found: error

# Step 2 of #175 (agent proposal): embed CPython without main from the node main thread, as
# the library does. A separate step after the upload, so a probe failure or hang does not
# lose the archive. Node only: node uses a real Atomics.wait on the main thread, a browser
# busy-waits, so this says nothing about the browser.
- name: Embedding probe (no main)
run: |
root="$(ls -d "$FT_WORK"/wasm-check/python-*t-wasm32-emscripten | head -n 1)"
[ -d "$root" ] || { echo "no unpacked archive under $FT_WORK/wasm-check"; exit 1; }
log="$FT_WORK/embed-probe.log"
set +e
timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs "$root" > "$log" 2>&1
rc=$?
set -e
cat "$log"
if [ "$rc" -ne 0 ]; then
last="$(grep -E '^\[[a-z0-9]+\]' "$log" | tail -n 1)"
echo "::error::embedding probe exited $rc (124 = timed out, a hang); last check printed: ${last:-none}"
exit "$rc"
fi
grep -q '^FT-WASM-EMBED-PROBE-OK$' "$log"
38 changes: 38 additions & 0 deletions docs/platforms/free-threaded-wasm.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,44 @@ The script stops at the first failed check, in this order:
The archive is uploaded as the workflow artifact `python-<version>t-wasm32-emscripten`. No
release is published or changed.

## Embedding probe (step 2 of #175)

Agent proposal toward #175 step 2, not a decision. It answers, on CI, whether the library's way
of using CPython (load `python.mjs` as a factory with `noInitialRun`, never call `main`, call
exports from the JS main thread) works against this pthreads build.

`build-wasm.sh` now appends `wasmExports,wasmMemory,wasmTable` to every
`-sEXPORTED_RUNTIME_METHODS` list in the host Makefile before `make-host` (the same edit as
`patch_makefile_reachability` in `python-multiplatform/scripts/wasm/build-cpython.sh`, as a
regex because the 3.15 Makefile text was not checked) and asserts the result. If the Makefile
has a `-sEXPORTED_FUNCTIONS=` list it also ensures `_malloc,_free,_Py_Version` are in it. A
missing list is only logged: whether 3.15 still exports C symbols through `-sMAIN_MODULE`
(as 3.14 does) is unverified, and the probe fails loudly if an export it needs is absent.

The workflow step `Embedding probe (no main)` runs after the archive upload, so a probe
failure or hang does not lose the archive:
`timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs <unpacked archive>`. It prints one
`[x] PASS|FAIL` line per check and `FT-WASM-EMBED-PROBE-OK` only if all pass; on a timeout
(exit 124) the step reports the last check line printed.

- `a`: the factory loads with `noInitialRun`, `Module.wasmMemory.buffer` is a
`SharedArrayBuffer` (page count printed).
- `b`: two hand-assembled modules import `intrinsics.memory` (the name Kotlin/Wasm uses).
A1 imports a non-shared memory and must fail with `LinkError`; A2 imports a shared one with
maximum 65536 pages and must instantiate, and `A2.load(&Py_Version)` must equal the value
read through a view of `wasmMemory`. A2 passing keeps the post-link import patch (investigation
options 2a/2b) open; A1 not failing would mean the blocker is not real.
- `c`: `Py_Initialize`, `PyEval_SaveThread`, then `PyGILState_Ensure` and `PyRun_SimpleString`
(strings placed with the exported `malloc`) of a script that starts four threads which print
and read `os.py`, joins them, asserts `sys._is_gil_enabled()` is False, and runs
`gc.collect()` inside a worker thread. This exercises the main thread blocking in `join`
while Workers need it to service proxied syscalls.

Node only. Node's main thread uses a real `Atomics.wait`; a browser's main thread busy-waits
and services proxied calls, so a pass here does not show the browser works, and a Node deadlock
may not occur in a browser. It does not involve Kotlin at all (no upcalls, no proxies, no
Kotlin-side memory import).

## Not verified

- The workflow has not run yet. Whether 3.15.0 configures, compiles and links with
Expand Down
Loading