|
| 1 | +// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal). |
| 2 | +// |
| 3 | +// Usage: node embed-probe.mjs <archive root> |
| 4 | +// |
| 5 | +// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory |
| 6 | +// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the |
| 7 | +// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final |
| 8 | +// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller |
| 9 | +// wraps this in `timeout` and reads the last "[x]" line it printed. |
| 10 | +// |
| 11 | +// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and |
| 12 | +// services proxied calls, so this probe says nothing about the browser. |
| 13 | +import fs from "node:fs"; |
| 14 | +import path from "node:path"; |
| 15 | +import { pathToFileURL } from "node:url"; |
| 16 | + |
| 17 | +const root = path.resolve(process.argv[2] ?? ""); |
| 18 | +if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) { |
| 19 | + console.log(`[setup] FAIL: usage: node embed-probe.mjs <archive root containing python.mjs>, got '${process.argv[2]}'`); |
| 20 | + process.exit(2); |
| 21 | +} |
| 22 | + |
| 23 | +let failed = false; |
| 24 | +const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`); |
| 25 | +const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); }; |
| 26 | +const finish = () => { |
| 27 | + if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); } |
| 28 | + console.log("FT-WASM-EMBED-PROBE-OK"); |
| 29 | + process.exit(0); // the pthread Workers would otherwise keep the process alive |
| 30 | +}; |
| 31 | + |
| 32 | +// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a |
| 33 | +// JSPI-suspending syscall import would throw SuspendError. |
| 34 | +delete WebAssembly.promising; |
| 35 | +delete WebAssembly.Suspending; |
| 36 | + |
| 37 | +const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d)); |
| 38 | +if (!libDir) { fail("a", `no lib/python3.<minor>t in ${root}`); finish(); } |
| 39 | +const minor = libDir.replace(/^python/, ""); |
| 40 | + |
| 41 | +// ---- a. load the factory without main ----------------------------------------------------------- |
| 42 | +let M; |
| 43 | +try { |
| 44 | + const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href); |
| 45 | + M = await EmscriptenModule({ |
| 46 | + noInitialRun: true, |
| 47 | + thisProgram: `${root}/bin/python${minor}`, |
| 48 | + arguments: [], |
| 49 | + preRun(Module) { |
| 50 | + globalThis.Module = Module; |
| 51 | + // Minimum of node_entry.mjs: mount the host filesystem (the archive is read through |
| 52 | + // NODEFS) except the three directories Emscripten owns, then set the environment. |
| 53 | + for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) { |
| 54 | + Module.FS.mkdirTree(dir); |
| 55 | + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); |
| 56 | + } |
| 57 | + Module.FS.chdir(root); |
| 58 | + Object.assign(Module.ENV, process.env); |
| 59 | + delete Module.ENV.PATH; |
| 60 | + Module.ENV.PYTHONHOME = root; |
| 61 | + }, |
| 62 | + }); |
| 63 | + const mem = M.wasmMemory; |
| 64 | + if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); |
| 65 | + if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); |
| 66 | + if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer"); |
| 67 | + pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`); |
| 68 | +} catch (e) { |
| 69 | + fail("a", String(e && e.stack || e)); |
| 70 | + finish(); |
| 71 | +} |
| 72 | +const E = M.wasmExports; |
| 73 | +// A fresh view each time: a shared memory's old buffer keeps its old length after growth. |
| 74 | +const u8 = () => new Uint8Array(M.wasmMemory.buffer); |
| 75 | +const u32 = () => new Uint32Array(M.wasmMemory.buffer); |
| 76 | +const exported = (name) => { |
| 77 | + let v = E[name]; |
| 78 | + if (v === undefined) v = M["_" + name]; |
| 79 | + if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`); |
| 80 | + return v; |
| 81 | +}; |
| 82 | +const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; }; |
| 83 | + |
| 84 | +// ---- b. memory import checks A1 / A2 ------------------------------------------------------------ |
| 85 | +// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and |
| 86 | +// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2 |
| 87 | +// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit). |
| 88 | +const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; |
| 89 | +const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f]; |
| 90 | +const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02]; |
| 91 | +const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00, |
| 92 | + 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b]; |
| 93 | +const mod = (limits) => new Uint8Array([...head, ...typeSec, |
| 94 | + 0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]); |
| 95 | +const A1 = mod([0x00, 0x00]); |
| 96 | +const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]); |
| 97 | +try { |
| 98 | + let a1; |
| 99 | + try { |
| 100 | + new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } }); |
| 101 | + a1 = "instantiated"; |
| 102 | + } catch (e) { |
| 103 | + a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`; |
| 104 | + if (a1 === "LinkError") console.log(` A1 message: ${e.message}`); |
| 105 | + } |
| 106 | + if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`); |
| 107 | + const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } }); |
| 108 | + const addr = addrOf("Py_Version"); |
| 109 | + const viaLoad = a2.exports.load(addr) >>> 0; |
| 110 | + const viaHeap = u32()[addr >>> 2]; |
| 111 | + if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`); |
| 112 | + pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`); |
| 113 | +} catch (e) { |
| 114 | + fail("b", String(e && e.stack || e)); |
| 115 | +} |
| 116 | + |
| 117 | +// ---- c. embedding without main, threads, gc in a worker ----------------------------------------- |
| 118 | +const cstr = (s) => { |
| 119 | + const bytes = Buffer.from(s + "\0"); |
| 120 | + const p = E.malloc(bytes.length); |
| 121 | + if (!p) throw new Error("malloc returned 0"); |
| 122 | + u8().set(bytes, p); |
| 123 | + return p; |
| 124 | +}; |
| 125 | +const script = ` |
| 126 | +import gc, os, sys, threading |
| 127 | +assert sys._is_gil_enabled() is False, "GIL is enabled at the start" |
| 128 | +sizes = {} |
| 129 | +def reader(i): |
| 130 | + with open(os.__file__) as f: |
| 131 | + n = len(f.read()) |
| 132 | + print(f"embed-probe thread {i} read {n} chars of os.py", flush=True) |
| 133 | + sizes[i] = n |
| 134 | +threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)] |
| 135 | +for t in threads: |
| 136 | + t.start() |
| 137 | +for t in threads: |
| 138 | + t.join(timeout=120) |
| 139 | +alive = [t.name for t in threads if t.is_alive()] |
| 140 | +assert not alive, f"threads still running: {alive}" |
| 141 | +assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes |
| 142 | +assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads" |
| 143 | +collected = [] |
| 144 | +gc_thread = threading.Thread(target=lambda: collected.append(gc.collect())) |
| 145 | +gc_thread.start() |
| 146 | +gc_thread.join(timeout=120) |
| 147 | +assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish" |
| 148 | +print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True) |
| 149 | +assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc" |
| 150 | +`; |
| 151 | +try { |
| 152 | + E.Py_Initialize(); |
| 153 | + pass("c1", "Py_Initialize returned"); |
| 154 | + const tstate = E.PyEval_SaveThread(); |
| 155 | + pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`); |
| 156 | + const gil = E.PyGILState_Ensure(); |
| 157 | + const code = cstr(script); |
| 158 | + console.log(" running the thread script (a hang here is caught by the caller's timeout)"); |
| 159 | + const rc = E.PyRun_SimpleString(code); |
| 160 | + E.free(code); |
| 161 | + E.PyGILState_Release(gil); |
| 162 | + if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`); |
| 163 | + pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned"); |
| 164 | +} catch (e) { |
| 165 | + fail("c", String(e && e.stack || e)); |
| 166 | +} |
| 167 | +finish(); |
0 commit comments