diff --git a/.github/scripts/ft-mobile/build-android.sh b/.github/scripts/ft-mobile/build-android.sh index cc5ced80d..cb9500cf8 100755 --- a/.github/scripts/ft-mobile/build-android.sh +++ b/.github/scripts/ft-mobile/build-android.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Build free-threaded CPython (--disable-gil) for aarch64-linux-android with -# CPython's own tooling (Platforms/Android). Usage: build-android.sh [VERSION] +# CPython's own tooling (Platforms/Android). Usage: build-android.sh [VERSION] [HOST] +# HOST is aarch64-linux-android (default) or x86_64-linux-android. # Needs ANDROID_HOME (SDK with cmdline-tools/latest) and a JDK 17, 21 or 25. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" @@ -8,7 +9,12 @@ here="$(cd "$(dirname "$0")" && pwd)" source "$here/common.sh" PY_VERSION="${1:-3.15.0}" -HOST=aarch64-linux-android +HOST="${2:-aarch64-linux-android}" +case "$HOST" in + aarch64-linux-android) elf_arch="ARM aarch64" ;; + x86_64-linux-android) elf_arch="x86-64" ;; + *) echo "unsupported HOST: $HOST" >&2; exit 2 ;; +esac : "${ANDROID_HOME:?ANDROID_HOME must point at the Android SDK}" src="$(fetch_source "$PY_VERSION")" @@ -55,9 +61,9 @@ require_gil_disabled "$check/prefix/include/python${minor}t/pyconfig.h" lib="$(ls "$check"/prefix/lib/libpython${minor}t*.so* 2>/dev/null | head -n 1 || true)" [ -n "$lib" ] || die "no libpython${minor}t*.so in the package" log "ok: $lib" -file "$lib" | tee /dev/stderr | grep -q 'ARM aarch64' || die "$lib is not an aarch64 ELF" +file "$lib" | tee /dev/stderr | grep -q "$elf_arch" || die "$lib is not an $elf_arch ELF" [ -d "$check/prefix/lib/python${minor}t" ] || die "no lib/python${minor}t in the package" cp "$archive" "$FT_OUT/" -echo "Android archive:" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +echo "Android archive ($HOST):" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" record_sha256 "$FT_OUT/$(basename "$archive")" diff --git a/.github/scripts/ft-wasm/build-wasm.sh b/.github/scripts/ft-wasm/build-wasm.sh new file mode 100755 index 000000000..5e9e86cff --- /dev/null +++ b/.github/scripts/ft-wasm/build-wasm.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Build free-threaded CPython (--disable-gil) with pthreads for wasm32-emscripten, +# with CPython's own driver (Platforms/emscripten). Step one of #175: a build +# experiment. Usage: build-wasm.sh [VERSION] +# +# Environment: +# FT_WORK scratch directory (default .tmp/ft-wasm) +# FT_OUT receives the archive the workflow uploads (default .tmp/ft-wasm-out) +# FT_WASM_EMSDK_VERSION Emscripten to install; empty = the version CPython itself pins +# (Platforms/emscripten/config.toml, 4.0.19 in 3.15.0) +# FT_WASM_TRAMPOLINE_PATCH apply (default) | skip; see patch_trampoline below +# +# Needs: a C toolchain for the native build Python, git, curl, node on PATH +# (CPython 3.15.0 pins node 24 in config.toml), python3 >= 3.11 to run the driver. +# +# This is a separate script, not a mode of python-multiplatform/scripts/wasm/build-cpython.sh: +# that script is pinned to CPython 3.14.2 / Emscripten 5.0.3 / pyemscripten_2026_0, clones a +# pinned git commit, writes into .caches/ and stages the runtime the wasmJs suite runs against. +# This job builds a different CPython (3.15.0 tarball) with a different Emscripten and must not +# touch that runtime. The flags below are the #175 investigation's proposal, not a decision. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" + +export FT_WORK="${FT_WORK:-$PWD/.tmp/ft-wasm}" +export FT_OUT="${FT_OUT:-$PWD/.tmp/ft-wasm-out}" +# fetch_source, require_gil_disabled, record_sha256, py_minor +# shellcheck source-path=SCRIPTDIR source=../ft-mobile/common.sh +source "$here/../ft-mobile/common.sh" +log() { echo "[ft-wasm] $*" >&2; } +die() { echo "[ft-wasm] ERROR: $*" >&2; exit 1; } + +PY_VERSION="${1:-3.15.0}" +TRAMPOLINE_PATCH="${FT_WASM_TRAMPOLINE_PATCH:-apply}" +minor="$(py_minor "$PY_VERSION")" +HOST=wasm32-emscripten + +# Compile and link flags (agent proposal from the #175 investigation, not confirmed). +# The driver hardcodes CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 and appends user args after +# it; autoconf keeps the last assignment, so the full string is repeated here. +# -fwasm-exceptions -sSUPPORT_LONGJMP=wasm same unwinding ABI as build-cpython.sh +# -pthread atomics + shared memory (configure also adds it +# to CFLAGS_NODIST for --enable-wasm-pthreads) +# -sPTHREAD_POOL_SIZE=8 pre-created Workers: Thread.start() blocks until +# the thread runs, and a Worker cannot start while +# the JS thread that asked for it is blocked +# -sMAXIMUM_MEMORY=2GB shared memory needs a declared maximum +ABI_FLAGS="-fwasm-exceptions -sSUPPORT_LONGJMP=wasm" +HOST_CFLAGS="-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 $ABI_FLAGS -pthread" +HOST_LDFLAGS="$ABI_FLAGS -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" + +src="$(fetch_source "$PY_VERSION")" +cd "$src" +[ -d Platforms/emscripten ] || die "no Platforms/emscripten in CPython $PY_VERSION (3.15+ expected)" + +# ---- Emscripten ----------------------------------------------------------- +# One emsdk under FT_WORK. The driver is then run without --emsdk-cache, so it uses the +# activated environment ("Build will use EMSDK from current environment"). +pinned="$(sed -n 's/^emscripten-version = "\(.*\)"$/\1/p' Platforms/emscripten/config.toml)" +[ -n "$pinned" ] || die "no emscripten-version in Platforms/emscripten/config.toml" +EMSDK_VERSION="${FT_WASM_EMSDK_VERSION:-$pinned}" +log "Emscripten pinned by CPython $PY_VERSION: $pinned; using: $EMSDK_VERSION" +emsdk="$FT_WORK/emsdk" +if [ ! -x "$emsdk/emsdk" ]; then + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$emsdk" +fi +"$emsdk/emsdk" install "$EMSDK_VERSION" +"$emsdk/emsdk" activate "$EMSDK_VERSION" +# shellcheck disable=SC1091 +EMSDK_QUIET=1 source "$emsdk/emsdk_env.sh" +emcc --version > "$FT_WORK/emcc-version.txt" +emcc_line="$(head -n 1 "$FT_WORK/emcc-version.txt")" +log "$emcc_line" +case "$emcc_line" in *" $EMSDK_VERSION "*) ;; *) die "emcc is not $EMSDK_VERSION" ;; esac +node --version + +# ---- patch: call trampoline under shared memory --------------------------- +# Python/emscripten_trampoline.c instantiates a wasm-gc trampoline module in pre-run. That +# module is compiled with -Wl,--import-memory and no -pthread (Makefile.pre.in), so it imports +# a NON-shared env.memory; giving it the shared wasmMemory of a pthread build is a link error, +# and the `new WebAssembly.Instance` call is outside the try block, so startup throws. +# Pyodide #237 also reports that the addFunction'ed pointer exists only in the main thread's +# table ("table index is out of bounds" on the first call from a spawned thread). +# Minimal fix: when memory is shared, return 0, which selects the existing JS trampoline +# (_PyEM_TrampolineCall_JS, wasmTable.get(func) on the calling thread). The edit is asserted, +# so an upstream change fails the build instead of silently skipping the patch. +# FT_WASM_TRAMPOLINE_PATCH=skip leaves the source alone, to observe the unpatched failure. +patch_trampoline() { + python3 - Python/emscripten_trampoline.c <<'PATCH' +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() +marker = "pmp-175: shared memory" +if marker not in t: + old = "function getPyEMTrampolinePtr() {\n" + new = (old + + " // " + marker + ": the wasm-gc trampoline imports non-shared memory and its\n" + " // addFunction pointer is valid only on this thread; use the JS trampoline.\n" + " if (typeof SharedArrayBuffer !== \"undefined\" && wasmMemory.buffer instanceof SharedArrayBuffer) {\n" + " return 0;\n" + " }\n") + assert t.count(old) == 1, "getPyEMTrampolinePtr changed shape; re-check the patch" + p.write_text(t.replace(old, new)) + print("patched Python/emscripten_trampoline.c (shared memory -> JS trampoline)") +PATCH +} +case "$TRAMPOLINE_PATCH" in + apply) patch_trampoline ;; + skip) log "trampoline patch skipped (FT_WASM_TRAMPOLINE_PATCH=skip)" ;; + *) die "FT_WASM_TRAMPOLINE_PATCH must be apply or skip, got '$TRAMPOLINE_PATCH'" ;; +esac + +# ---- native build Python (free-threaded too, as in ft-mobile) ------------- +python3 --version +python3 Platforms/emscripten configure-build-python -- --disable-gil +python3 Platforms/emscripten make-build-python +python3 Platforms/emscripten pythoninfo-build +cross-build/build/python -c 'import sys, sysconfig; assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1; print("build python free-threaded:", sys.version)' + +# ---- libffi and mpdec with -pthread --------------------------------------- +# The driver builds them with CFLAGS="-O2 -fPIC -DWASM_BIGINT" (make_libffi.sh) and +# CFLAGS=-fPIC (mpdec). emcc appends EMCC_CFLAGS to every invocation, so -pthread reaches +# both without editing the driver. Checked below by the objects' target_features section. +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-libffi +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-mpdec + +prefix="cross-build/$HOST/prefix" +for lib in "$prefix/lib/libffi.a" "$prefix/lib/libmpdec.a"; do + [ -f "$lib" ] || die "no $lib" + # wasm target_features entries are : "+", 7, "atomics". + python3 - "$lib" <<'CHECK' || die "$lib was not compiled with atomics (-pthread)" +import sys +data = open(sys.argv[1], "rb").read() +sys.exit(0 if b"+\x07atomics" in data else 1) +CHECK + log "ok: $lib has +atomics" +done + +# ---- host configure ------------------------------------------------------- +python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=$HOST_CFLAGS" \ + "LDFLAGS=$HOST_LDFLAGS" \ + --disable-gil --enable-wasm-pthreads + +hostdir="cross-build/$HOST/build/python" +require_gil_disabled "$hostdir/pyconfig.h" +# configure adds -sUSE_PTHREADS to LDFLAGS_NODIST (CONFIGURE_LDFLAGS_NODIST in the +# Makefile) only when --enable-wasm-pthreads took effect. +grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \ + || die "configure did not enable wasm pthreads (no -sUSE_PTHREADS in LDFLAGS_NODIST)" +grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t" +log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t" + +# ---- patch: JS reachability for the embedding probe ----------------------- +# Same substitution as patch_makefile_reachability in python-multiplatform/scripts/wasm/build-cpython.sh: +# the library reaches CPython without main through Module.wasmExports / Module.wasmMemory, which +# the stock link line does not expose (3.15 default: FS,callMain,ENV,HEAPU32,TTY). Written as a +# regex over the whole comma list rather than a fixed string, because the 3.15 Makefile was not +# available when this was written. The result is asserted, so a changed link line fails the +# build here instead of producing an archive the probe cannot use. +# The C functions: CPython's LINKFORSHARED lists -sEXPORTED_FUNCTIONS explicitly +# (3.14: _main,_Py_Version,__PyRuntime,...) next to -sMAIN_MODULE. Whether 3.15 still does is +# unverified, so _malloc,_free,_Py_Version are appended to that list when it exists (harmless if +# MAIN_MODULE already exports everything); the probe then checks the exports it needs and +# fails loudly if one is missing. +patch_makefile_exports() { + python3 - "$hostdir/Makefile" <<'PATCH' +import re +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() + +def add_runtime_methods(m): + items = m.group(2).split(",") + for extra in ("wasmExports", "wasmMemory", "wasmTable"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, n = re.subn(r"(-sEXPORTED_RUNTIME_METHODS=)(\S+)", add_runtime_methods, t) +assert n >= 1, "no -sEXPORTED_RUNTIME_METHODS= in the host Makefile; the link line changed shape, re-check the patch" +for m in re.finditer(r"-sEXPORTED_RUNTIME_METHODS=(\S+)", t): + methods = m.group(1).split(",") + assert all(x in methods for x in ("wasmExports", "wasmMemory", "wasmTable")), "wasmExports/wasmMemory/wasmTable missing after the patch: " + m.group(0) +print(f"Makefile: wasmExports,wasmMemory,wasmTable added to {n} -sEXPORTED_RUNTIME_METHODS list(s)") + +def add_functions(m): + items = m.group(2).split(",") + for extra in ("_malloc", "_free", "_Py_Version"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, k = re.subn(r"(-sEXPORTED_FUNCTIONS=)(\S+)", add_functions, t) +if k: + print(f"Makefile: _malloc,_free,_Py_Version ensured in {k} -sEXPORTED_FUNCTIONS list(s)") +else: + print("Makefile: NOTE no -sEXPORTED_FUNCTIONS= found; relying on the link exporting C symbols another way (the probe checks)") +p.write_text(t) +for line in t.splitlines(): + if "-sEXPORTED_RUNTIME_METHODS" in line or "-sEXPORTED_FUNCTIONS" in line or "-sMAIN_MODULE" in line: + print("Makefile link line:", line.strip()[:2000]) +PATCH +} +patch_makefile_exports + +# ---- host build ----------------------------------------------------------- +# make-host also runs `python.sh --version`, so a startup failure (for example the trampoline +# instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step. +python3 Platforms/emscripten make-host + +# ---- verification --------------------------------------------------------- +# python.wasm must use shared memory with a declared maximum. +node - "$hostdir/python.wasm" <<'NODE' || die "python.wasm has no shared memory" +const b = require("fs").readFileSync(process.argv[2]); +const leb = (i) => { let r = 0, s = 0; for (;;) { const x = b[i++]; r += (x & 0x7f) * 2 ** s; s += 7; if (!(x & 0x80)) return [r, i]; } }; +const str = (i) => { let n; [n, i] = leb(i); return [b.slice(i, i + n).toString(), i + n]; }; +const limits = (i) => { const flags = b[i++]; let min, max = null; [min, i] = leb(i); if (flags & 1) [max, i] = leb(i); return [{ flags, min, max }, i]; }; +let i = 8, mem = null; +while (i < b.length && !mem) { + const id = b[i++]; let size; [size, i] = leb(i); const end = i + size; + if (id === 2) { let n, j; [n, j] = leb(i); + for (let k = 0; k < n && !mem; k++) { let m, f; [m, j] = str(j); [f, j] = str(j); const kind = b[j++]; + if (kind === 0) { [, j] = leb(j); } + else if (kind === 1) { j++; [, j] = limits(j); } + else if (kind === 2) { let l; [l, j] = limits(j); mem = { where: `import ${m}.${f}`, ...l }; } + else if (kind === 3) { j += 2; } + else if (kind === 4) { j++; [, j] = leb(j); } + else throw new Error(`unknown import kind ${kind}`); } } + if (id === 5) { let j; [, j] = leb(i); const [l] = limits(j); mem = { where: "defined", ...l }; } + i = end; +} +if (!mem) { console.error("no memory found"); process.exit(1); } +const shared = (mem.flags & 2) !== 0; +console.log(`python.wasm memory: ${mem.where} shared=${shared} min=${mem.min} max=${mem.max} pages`); +process.exit(shared && mem.max !== null ? 0 : 1); +NODE + +# The interpreter itself: GIL off at run time, four real threads. +run_smoke() { + local label="$1"; shift + local out="$FT_WORK/smoke-$label.log" rc + set +e + timeout 600 "$@" "$here/smoke.py" > "$out" 2>&1 + rc=$? + set -e + cat "$out" + [ "$rc" -eq 0 ] || die "smoke test ($label) exited $rc (124 = timed out, a thread start may have deadlocked)" + grep -q '^FT-WASM-SMOKE-OK ' "$out" || die "smoke test ($label) did not print FT-WASM-SMOKE-OK" + log "ok: smoke test ($label)" +} +run_smoke build-tree "$PWD/$hostdir/python.sh" + +# ---- archive -------------------------------------------------------------- +# The driver has no install or package step for Emscripten, so the archive is assembled here. +# The stdlib directory is named by the built interpreter's own sysconfig, so a non-FT build +# (python3.15) fails the name check instead of being renamed. +stdlib_name="$(timeout 300 "$PWD/$hostdir/python.sh" -c 'import os, sysconfig; print(os.path.basename(sysconfig.get_path("stdlib")))' | tail -n 1)" +[ "$stdlib_name" = "python${minor}t" ] || die "interpreter's stdlib dir is '$stdlib_name', expected python${minor}t" +log "ok: interpreter reports lib/$stdlib_name" + +name="python-${PY_VERSION}t-$HOST" +stage="$FT_WORK/stage/$name" +rm -rf "$FT_WORK/stage"; mkdir -p "$stage/lib/$stdlib_name/lib-dynload" "$stage/include/$stdlib_name" +cp "$hostdir/python.mjs" "$hostdir/python.wasm" "$stage/" +cp "$hostdir/node_entry.mjs" "$hostdir/streams.mjs" "$stage/" +# Everything but the test suite; the zip-and-omit selection is left to whoever stages it. +tar -C Lib --exclude='./test' --exclude='__pycache__' -cf - . | tar -C "$stage/lib/$stdlib_name" -xf - +cp "$hostdir/$(cat "$hostdir/pybuilddir.txt")"/_sysconfigdata_*.py "$stage/lib/$stdlib_name/" +touch "$stage/lib/$stdlib_name/lib-dynload/.empty" +cp -R Include/. "$stage/include/$stdlib_name/" +cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/" +{ + echo "CPython $PY_VERSION, --disable-gil --enable-wasm-pthreads, $HOST" + echo "emcc: $emcc_line" + echo "CFLAGS=$HOST_CFLAGS" + echo "LDFLAGS=$HOST_LDFLAGS" + echo "trampoline patch: $TRAMPOLINE_PATCH" + echo "link patch: wasmExports,wasmMemory,wasmTable in EXPORTED_RUNTIME_METHODS (embedding probe)" + echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)" +} > "$stage/BUILD-INFO.txt" + +archive="$FT_OUT/$name.tar.gz" +mkdir -p "$FT_OUT" +tar -C "$FT_WORK/stage" -czf "$archive" "$name" + +# Re-check the archive itself, unpacked somewhere else, and run it with PYTHONHOME pointing +# at it (node_entry.mjs copies process.env into the Emscripten ENV, and NODEFS mounts /). +check="$FT_WORK/wasm-check" +rm -rf "$check"; mkdir -p "$check" +tar -xzf "$archive" -C "$check" +root="$check/$name" +require_gil_disabled "$root/include/python${minor}t/pyconfig.h" +[ -d "$root/lib/python${minor}t" ] || die "no lib/python${minor}t in the archive" +[ -f "$root/lib/python${minor}t/os.py" ] || die "no lib/python${minor}t/os.py in the archive" +ls "$root/lib/python${minor}t"/_sysconfigdata_*.py > /dev/null || die "no _sysconfigdata in the archive" +nodeflags="" +[ "$(node -p 'process.versions.node.split(".")[0]')" = "24" ] && nodeflags="--experimental-wasm-jspi" +# shellcheck disable=SC2086 +run_smoke archive env PYTHONHOME="$root" node $nodeflags "$root/node_entry.mjs" \ + --this-program="$root/bin/python${minor}t" + +echo "wasm32-emscripten archive (Emscripten $EMSDK_VERSION, trampoline patch: $TRAMPOLINE_PATCH):" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +record_sha256 "$archive" diff --git a/.github/scripts/ft-wasm/embed-probe.mjs b/.github/scripts/ft-wasm/embed-probe.mjs new file mode 100644 index 000000000..375d22396 --- /dev/null +++ b/.github/scripts/ft-wasm/embed-probe.mjs @@ -0,0 +1,167 @@ +// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal). +// +// Usage: node embed-probe.mjs +// +// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory +// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the +// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final +// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller +// wraps this in `timeout` and reads the last "[x]" line it printed. +// +// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and +// services proxied calls, so this probe says nothing about the browser. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const root = path.resolve(process.argv[2] ?? ""); +if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) { + console.log(`[setup] FAIL: usage: node embed-probe.mjs , got '${process.argv[2]}'`); + process.exit(2); +} + +let failed = false; +const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`); +const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); }; +const finish = () => { + if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); } + console.log("FT-WASM-EMBED-PROBE-OK"); + process.exit(0); // the pthread Workers would otherwise keep the process alive +}; + +// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a +// JSPI-suspending syscall import would throw SuspendError. +delete WebAssembly.promising; +delete WebAssembly.Suspending; + +const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d)); +if (!libDir) { fail("a", `no lib/python3.t in ${root}`); finish(); } +const minor = libDir.replace(/^python/, ""); + +// ---- a. load the factory without main ----------------------------------------------------------- +let M; +try { + const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href); + M = await EmscriptenModule({ + noInitialRun: true, + thisProgram: `${root}/bin/python${minor}`, + arguments: [], + preRun(Module) { + globalThis.Module = Module; + // Minimum of node_entry.mjs: mount the host filesystem (the archive is read through + // NODEFS) except the three directories Emscripten owns, then set the environment. + for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } + Module.FS.chdir(root); + Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; + Module.ENV.PYTHONHOME = root; + }, + }); + const mem = M.wasmMemory; + if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer"); + pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`); +} catch (e) { + fail("a", String(e && e.stack || e)); + finish(); +} +const E = M.wasmExports; +// A fresh view each time: a shared memory's old buffer keeps its old length after growth. +const u8 = () => new Uint8Array(M.wasmMemory.buffer); +const u32 = () => new Uint32Array(M.wasmMemory.buffer); +const exported = (name) => { + let v = E[name]; + if (v === undefined) v = M["_" + name]; + if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`); + return v; +}; +const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; }; + +// ---- b. memory import checks A1 / A2 ------------------------------------------------------------ +// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and +// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2 +// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit). +const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; +const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f]; +const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02]; +const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00, + 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b]; +const mod = (limits) => new Uint8Array([...head, ...typeSec, + 0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]); +const A1 = mod([0x00, 0x00]); +const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]); +try { + let a1; + try { + new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } }); + a1 = "instantiated"; + } catch (e) { + a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`; + if (a1 === "LinkError") console.log(` A1 message: ${e.message}`); + } + if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`); + const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } }); + const addr = addrOf("Py_Version"); + const viaLoad = a2.exports.load(addr) >>> 0; + const viaHeap = u32()[addr >>> 2]; + if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`); + pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`); +} catch (e) { + fail("b", String(e && e.stack || e)); +} + +// ---- c. embedding without main, threads, gc in a worker ----------------------------------------- +const cstr = (s) => { + const bytes = Buffer.from(s + "\0"); + const p = E.malloc(bytes.length); + if (!p) throw new Error("malloc returned 0"); + u8().set(bytes, p); + return p; +}; +const script = ` +import gc, os, sys, threading +assert sys._is_gil_enabled() is False, "GIL is enabled at the start" +sizes = {} +def reader(i): + with open(os.__file__) as f: + n = len(f.read()) + print(f"embed-probe thread {i} read {n} chars of os.py", flush=True) + sizes[i] = n +threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running: {alive}" +assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes +assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads" +collected = [] +gc_thread = threading.Thread(target=lambda: collected.append(gc.collect())) +gc_thread.start() +gc_thread.join(timeout=120) +assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish" +print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True) +assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc" +`; +try { + E.Py_Initialize(); + pass("c1", "Py_Initialize returned"); + const tstate = E.PyEval_SaveThread(); + pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`); + const gil = E.PyGILState_Ensure(); + const code = cstr(script); + console.log(" running the thread script (a hang here is caught by the caller's timeout)"); + const rc = E.PyRun_SimpleString(code); + E.free(code); + E.PyGILState_Release(gil); + if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`); + pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned"); +} catch (e) { + fail("c", String(e && e.stack || e)); +} +finish(); diff --git a/.github/scripts/ft-wasm/smoke.py b/.github/scripts/ft-wasm/smoke.py new file mode 100644 index 000000000..78c3758ce --- /dev/null +++ b/.github/scripts/ft-wasm/smoke.py @@ -0,0 +1,42 @@ +# Smoke test for the free-threaded wasm32-emscripten build (#175), run by build-wasm.sh +# under node. Prints one line starting with FT-WASM-SMOKE-OK, which the script greps for; +# any failed assertion exits non-zero instead. +import sys +import sysconfig +import threading + +assert sys.platform == "emscripten", sys.platform +assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1, "not a free-threaded build" +assert sys._is_gil_enabled() is False, "the GIL is enabled at run time" + +N_THREADS = 4 +N_ITER = 200_000 +results = [None] * N_THREADS + + +def work(i): + total = 0 + for k in range(N_ITER): + total += k * (i + 1) + results[i] = (threading.get_ident(), total) + + +threads = [threading.Thread(target=work, args=(i,)) for i in range(N_THREADS)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running after join: {alive}" + +expected = [(i + 1) * N_ITER * (N_ITER - 1) // 2 for i in range(N_THREADS)] +got = [r[1] if r else None for r in results] +assert got == expected, f"results {got} != {expected}" +idents = {r[0] for r in results} +assert len(idents) == N_THREADS, f"expected {N_THREADS} distinct thread idents, got {idents}" +assert threading.get_ident() not in idents, "a worker ran on the main thread" +# Importing threading must not have re-enabled the GIL. +assert sys._is_gil_enabled() is False, "the GIL was re-enabled during the run" + +print(f"FT-WASM-SMOKE-OK gil={sys._is_gil_enabled()} threads={N_THREADS} " + f"abiflags={sys.abiflags} version={sys.version.split()[0]}") diff --git a/.github/scripts/release/central-bundle.sh b/.github/scripts/release/central-bundle.sh new file mode 100755 index 000000000..84e0b9585 --- /dev/null +++ b/.github/scripts/release/central-bundle.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Builds the Maven Central deployment of python-multiplatform: every component, signed, in one zip. +# .github/scripts/release/central-bundle.sh [--throwaway-key] +# Run from the repository root. Reads MAVEN_GPG_PRIVATE_KEY and MAVEN_GPG_PASSPHRASE (Gradle signs +# in memory, gradle/central-publishing.gradle.kts). --throwaway-key makes a key for this run only, +# for a dry run that must prove signing works without the release key. Without either, the bundle is +# unsigned and the script says so; Central would refuse it. +# Writes build/central-staging (a Maven layout) and ; uploads nothing, see central-upload.sh. +set -euo pipefail +out="${1:?usage: central-bundle.sh [--throwaway-key]}" +mode="${2:-}" +root="$(pwd)" +staging="$root/build/central-staging" +version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + +# Not under the repository: gpg-agent's socket path must stay under about 100 characters. +verify_home="$(mktemp -d)" +chmod 700 "$verify_home" +trap 'rm -rf "$verify_home"' EXIT + +if [[ "$mode" == "--throwaway-key" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --passphrase '' --quick-generate-key \ + "python-multiplatform dry run " rsa3072 sign 1d + MAVEN_GPG_PRIVATE_KEY="$(GNUPGHOME="$verify_home" gpg --batch --pinentry-mode loopback --passphrase '' --armor --export-secret-keys)" + export MAVEN_GPG_PRIVATE_KEY MAVEN_GPG_PASSPHRASE="" +elif [[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --import <<< "$MAVEN_GPG_PRIVATE_KEY" +fi +signed=false +[[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]] && signed=true + +mkdir -p "$root/.cache" +rm -rf "$staging" +# Gradle reads the key from the environment; the plugin build is a separate build but inherits it. +./gradlew --console=plain publishAllToCentralStaging 2>&1 | tee "$root/.cache/central-gradle.log" || { echo "error: gradle failed (pipe status ${PIPESTATUS[0]})" >&2; exit 1; } + +# What Central checks first: every module has pom + jar/klib/zip, sources and javadoc where it is a +# jar publication, and (signed) a good .asc beside every file. +fail=0 +while IFS= read -r pom; do + dir="$(dirname "$pom")"; base="$(basename "$pom" .pom)" + ver="$(basename "$dir")"; art="$(basename "$(dirname "$dir")")" + [[ "$ver" == "$version" ]] || { echo "error: $pom is not at $version" >&2; fail=1; } + grep -q 'https://github.com/thisisthepy/python-multiplatform' "$pom" || { echo "error: $pom has no project url" >&2; fail=1; } + grep -q '' "$pom" && grep -q '' "$pom" && grep -q '' "$pom" || { echo "error: $pom lacks licenses, developers or scm" >&2; fail=1; } + if ! grep -q 'pom' "$pom"; then + [[ -e "$dir/$base-javadoc.jar" ]] || { echo "error: $art has no javadoc jar" >&2; fail=1; } + [[ -e "$dir/$base-sources.jar" ]] || { echo "error: $art has no sources jar" >&2; fail=1; } + fi +done < <(find "$staging" -name '*.pom') +# An AAR must carry the native libraries of this pythonVersion only. A Copy-staged jniLibs tree once +# kept libpython3.14.so beside libpython3.15.so after a version bump. libpython3.so (the stable-ABI +# stub) carries no minor version and is allowed. +py_mm="$(sed -nE 's/^pythonVersion=([0-9]+\.[0-9]+).*$/\1/p' gradle.properties)" +while IFS= read -r aar; do + stale="$(unzip -Z1 "$aar" | grep -E '(^|/)lib(multiplatform_)?python[0-9]+\.[0-9]+[^/]*\.so$' | grep -vE "(^|/)lib(multiplatform_)?python${py_mm//./\\.}(t)?\.so$" || true)" + [[ -z "$stale" ]] || { echo "error: $aar carries native libraries for another python than $py_mm:" >&2; echo "$stale" >&2; fail=1; } +done < <(find "$staging" -name '*.aar') +if $signed; then + while IFS= read -r f; do + case "$f" in *.asc|*.md5|*.sha1|*.sha256|*.sha512) continue ;; esac + [[ -e "$f.asc" ]] || { echo "error: $f is not signed" >&2; fail=1; continue; } + GNUPGHOME="$verify_home" gpg --batch --quiet --verify "$f.asc" "$f" 2>/dev/null || { echo "error: bad signature on $f" >&2; fail=1; } + done < <(find "$staging" -type f -not -name 'maven-metadata*') +else + echo "warning: no signing key, the bundle is unsigned and Central will refuse it" >&2 +fi +(( fail == 0 )) || exit 1 + +echo "== left out of the bundle (not built on this machine)" +grep -h "CENTRAL-SKIPPED" "$root/.cache/central-gradle.log" || echo "(none)" +echo "== coordinates in the bundle" +find "$staging" -name '*.pom' | while IFS= read -r pom; do + rel="${pom#"$staging"/}"; ver="$(basename "$(dirname "$rel")")"; art="$(basename "$(dirname "$(dirname "$rel")")")" + grp="$(dirname "$(dirname "$(dirname "$rel")")" | tr / .)" + echo "$grp:$art:$ver" +done | sort + +mkdir -p "$(dirname "$out")" +out="$(cd "$(dirname "$out")" && pwd)/$(basename "$out")" +rm -f "$out" +(cd "$staging" && zip -qr "$out" . -x 'maven-metadata*' '*/maven-metadata*') +echo "== bundle $out ($(find "$staging" -type f | wc -l | tr -d ' ') files)" diff --git a/.github/scripts/release/central-upload.sh b/.github/scripts/release/central-upload.sh new file mode 100755 index 000000000..39c3a85e1 --- /dev/null +++ b/.github/scripts/release/central-upload.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Uploads a bundle made by central_bundle.py to the Sonatype Central Portal and waits for its +# validation. +# +# Usage: central-upload.sh [USER_MANAGED|AUTOMATIC] +# +# Reads CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD, the two halves of a Central Portal +# user token (central.sonatype.com, View Account, Generate User Token). The Portal API takes +# them as a bearer token of base64(":"). +# +# USER_MANAGED (the default) stops after validation: the deployment waits in the Portal under +# Deployments until someone presses Publish or Drop, so a person looks at it before it becomes +# permanent. AUTOMATIC publishes as soon as validation passes. Either way nothing published to +# Central can be changed or removed afterwards. +# +# Exits 0 when the deployment is VALIDATED (USER_MANAGED) or PUBLISHING/PUBLISHED (AUTOMATIC), +# 1 when it FAILED, printing the Portal's errors, and 2 on a usage or network error. +set -euo pipefail + +API="https://central.sonatype.com/api/v1/publisher" + +bundle="${1:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +name="${2:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +type="${3:-USER_MANAGED}" +case "$type" in USER_MANAGED|AUTOMATIC) ;; *) echo "error: publishing type '$type'" >&2; exit 2 ;; esac +[[ -s "$bundle" ]] || { echo "error: no bundle at $bundle" >&2; exit 2; } +[[ -n "${CENTRAL_TOKEN_USERNAME:-}" && -n "${CENTRAL_TOKEN_PASSWORD:-}" ]] || { + echo "error: CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD must hold a Central Portal user token" >&2 + exit 2 +} + +token="$(printf '%s:%s' "$CENTRAL_TOKEN_USERNAME" "$CENTRAL_TOKEN_PASSWORD" | base64 | tr -d '\n')" +auth="Authorization: Bearer $token" + +encoded_name="$(python3 -I -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$name")" +echo "==> uploading $bundle as '$name' ($type)" +id="$(curl --fail-with-body --silent --show-error --retry 3 \ + --header "$auth" \ + --form "bundle=@$bundle" \ + "$API/upload?publishingType=$type&name=$encoded_name")" +echo "deployment $id" + +# Validation of a large bundle takes minutes; give it an hour. +for _ in $(seq 1 120); do + status="$(curl --fail-with-body --silent --show-error --retry 3 --request POST \ + --header "$auth" "$API/status?id=$id")" + state="$(python3 -I -c 'import json, sys; print(json.load(sys.stdin)["deploymentState"])' <<< "$status")" + echo "state $state" + case "$state" in + VALIDATED) + if [[ "$type" == USER_MANAGED ]]; then + echo "validated; publish or drop it at https://central.sonatype.com/publishing/deployments" + exit 0 + fi + ;; + PUBLISHING|PUBLISHED) + echo "publishing; it appears on https://repo1.maven.org/maven2/org/thisisthepy/ within about 30 minutes" + exit 0 + ;; + FAILED) + python3 -I -c 'import json, sys; print(json.dumps(json.load(sys.stdin).get("errors"), indent=2))' <<< "$status" >&2 + exit 1 + ;; + esac + sleep 30 +done +echo "error: deployment $id is still $state after an hour; see https://central.sonatype.com/publishing/deployments" >&2 +exit 2 diff --git a/.github/scripts/stubs/consumer.py b/.github/scripts/stubs/consumer.py index 99e4e61d8..dd7fbaa8a 100644 --- a/.github/scripts/stubs/consumer.py +++ b/.github/scripts/stubs/consumer.py @@ -28,6 +28,7 @@ from androidx.compose.material3 import Checkbox, Icon, Text from androidx.compose.ui import Modifier from androidx.compose.ui.graphics.vector import ImageVector +from org.thisisthepy.compose.material3 import Material3DesignSystem, material_design_system # A chain started from the class, the way Kotlin writes `Modifier.padding(16.dp)`. chain: Modifier = Modifier.fillMaxWidth().padding(16.0).size(24.0) @@ -43,6 +44,10 @@ assert_type(fill_max_width(chain), Modifier) pythonic_chain: Modifier = Modifier.fill_max_width().padding(16.0).fill_max_width(0.5) +# Issue #408: the fork's design systems are walked too, under their own Kotlin packages. +dark: Material3DesignSystem = material_design_system(True) +assert_type(material_design_system(is_dark=False), Material3DesignSystem) + checked = True diff --git a/.github/workflows/android-native.yml b/.github/workflows/android-native.yml index 3e77c4f17..c85f06d55 100644 --- a/.github/workflows/android-native.yml +++ b/.github/workflows/android-native.yml @@ -1,16 +1,9 @@ name: Android Native CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/.github/workflows/build-ft-mobile.yml b/.github/workflows/build-ft-mobile.yml index 0b9d05954..aca3f7525 100644 --- a/.github/workflows/build-ft-mobile.yml +++ b/.github/workflows/build-ft-mobile.yml @@ -1,18 +1,12 @@ name: Build free-threaded CPython for mobile -# Builds CPython with --disable-gil ("3.15t") for Android arm64 and iOS with +# Builds CPython with --disable-gil ("3.15t") for Android (aarch64 and x86_64) and iOS with # CPython's own tooling (Platforms/Android, Platforms/Apple). Manual only: the # builds take a long time and the archives are inputs to later work (#163). # See docs/platforms/free-threaded-mobile.md. on: - # Runs on develop when the build itself changes (workflow_dispatch needs the file on the - # default branch, main, which develop reaches only through a release). - push: - branches: [develop] - paths: - - ".github/workflows/build-ft-mobile.yml" - - ".github/scripts/ft-mobile/**" + # Started by hand only (owner, 2026-11-14: "개발 테스트 ci는 명시적으로 실행하기"). workflow_dispatch: inputs: python_version: @@ -22,12 +16,19 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # One development run per repository: a run in progress finishes, and a newer dispatch replaces + # the waiting one (owner, 2026-11-14: "제일 최신꺼만 도는게 맞지"). + group: dev-ci cancel-in-progress: false jobs: android: - name: Android aarch64 (free-threaded) + # Both ABIs the library packages (arm64-v8a, x86_64); x86_64 approved by the owner 2026-11-13 (#328). + name: Android ${{ matrix.host }} (free-threaded) + strategy: + fail-fast: false + matrix: + host: [aarch64-linux-android, x86_64-linux-android] runs-on: ubuntu-24.04 timeout-minutes: 120 env: @@ -48,11 +49,11 @@ jobs: libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev - name: Build - run: .github/scripts/ft-mobile/build-android.sh "${{ inputs.python_version || '3.15.0' }}" + run: .github/scripts/ft-mobile/build-android.sh "${{ inputs.python_version || '3.15.0' }}" "${{ matrix.host }}" - uses: actions/upload-artifact@v4 with: - name: python-${{ inputs.python_version || '3.15.0' }}t-android-aarch64 + name: python-${{ inputs.python_version || '3.15.0' }}t-${{ matrix.host }} path: ${{ env.FT_OUT }}/*.tar.gz if-no-files-found: error diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml new file mode 100644 index 000000000..25f2f6c58 --- /dev/null +++ b/.github/workflows/build-ft-wasm.yml @@ -0,0 +1,94 @@ +name: Build free-threaded CPython for wasm + +# Builds CPython with --disable-gil and pthreads ("3.15t") for wasm32-emscripten with +# CPython's own driver (Platforms/emscripten), then runs it under node with four threads. +# Step one of #175, a build experiment; job approved by the owner on 2026-11-13 ("#175 may +# add the new wasm 3.15t CI build job"). It does not touch wasm.yml or the runtime the +# wasmJs suite uses. See docs/platforms/free-threaded-wasm.md. +# +# Experiment choices (agent proposal, not decisions): Emscripten defaults to the version +# CPython itself pins (Platforms/emscripten/config.toml, 4.0.19 for 3.15.0), not the 5.0.3 +# of python-multiplatform/scripts/wasm/build-cpython.sh; the compile/link flags are the +# #175 investigation's; the call trampoline is patched to fall back to its JS path under +# shared memory (trampoline_patch=skip runs without it). + +on: + # Started by hand only (owner, 2026-11-14: "개발 테스트 ci는 명시적으로 실행하기"). + workflow_dispatch: + inputs: + python_version: + description: CPython version to build (a python.org source release, 3.15 or later) + default: "3.15.0" + required: true + type: string + emscripten_version: + description: Emscripten to use (empty = the version CPython pins in Platforms/emscripten/config.toml) + default: "" + required: false + type: string + trampoline_patch: + description: Call-trampoline shared-memory patch (skip = observe the unpatched build) + default: apply + required: true + type: choice + options: [apply, skip] + +concurrency: + # One development run per repository: a run in progress finishes, and a newer dispatch replaces + # the waiting one (owner, 2026-11-14: "제일 최신꺼만 도는게 맞지"). + group: dev-ci + cancel-in-progress: false + +jobs: + wasm: + name: wasm32-emscripten pthreads (free-threaded) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + FT_WORK: ${{ github.workspace }}/.tmp/ft-wasm + FT_OUT: ${{ github.workspace }}/.tmp/ft-wasm-out + FT_WASM_EMSDK_VERSION: ${{ inputs.emscripten_version || '' }} + FT_WASM_TRAMPOLINE_PATCH: ${{ inputs.trampoline_patch || 'apply' }} + steps: + - uses: actions/checkout@v4 + + # CPython 3.15.0 pins node 24 (Platforms/emscripten/config.toml, node-version). + - uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install build Python prerequisites + run: > + sudo apt-get update && sudo apt-get install -y + build-essential pkg-config libssl-dev zlib1g-dev libffi-dev + libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev + + - name: Build and verify + run: .github/scripts/ft-wasm/build-wasm.sh "${{ inputs.python_version || '3.15.0' }}" + + - uses: actions/upload-artifact@v4 + with: + name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten + path: ${{ env.FT_OUT }}/*.tar.gz + if-no-files-found: error + + # Step 2 of #175 (agent proposal): embed CPython without main from the node main thread, as + # the library does. A separate step after the upload, so a probe failure or hang does not + # lose the archive. Node only: node uses a real Atomics.wait on the main thread, a browser + # busy-waits, so this says nothing about the browser. + - name: Embedding probe (no main) + run: | + root="$(ls -d "$FT_WORK"/wasm-check/python-*t-wasm32-emscripten | head -n 1)" + [ -d "$root" ] || { echo "no unpacked archive under $FT_WORK/wasm-check"; exit 1; } + log="$FT_WORK/embed-probe.log" + set +e + timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs "$root" > "$log" 2>&1 + rc=$? + set -e + cat "$log" + if [ "$rc" -ne 0 ]; then + last="$(grep -E '^\[[a-z0-9]+\]' "$log" | tail -n 1)" + echo "::error::embedding probe exited $rc (124 = timed out, a hang); last check printed: ${last:-none}" + exit "$rc" + fi + grep -q '^FT-WASM-EMBED-PROBE-OK$' "$log" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..518fccb45 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: CI + +# The one development CI of this repository. It is started by hand only (owner, 2026-11-14: +# "개발 테스트 ci는 명시적으로 실행하기", "한 ci에서 최대한 여러개를 확인하기"), and one run checks +# every platform: each suite below is a reusable workflow, on by default. +# +# gh workflow run ci.yml --ref +# +# The concurrency group is shared with the CPython builds (build-ft-*.yml): a run in progress +# finishes, a newer dispatch replaces the waiting one, and nothing else runs beside it (owner: +# "제일 최신꺼만 도는게 맞지", "여러 ci를 한번에 돌리는거도 허용할 생각이 난 없어"). + +on: + workflow_dispatch: + inputs: + desktop: + description: "Desktop (JVM, macosArm64, linuxX64) and the generator fixtures" + type: boolean + default: true + android: + description: "Android and androidNative" + type: boolean + default: true + ios: + description: "iOS" + type: boolean + default: true + wasm: + description: "WasmJS" + type: boolean + default: true + typedpython: + description: "TypedPython gate and compiler" + type: boolean + default: true + race_mutant: + description: "TypedPython: also run the race_mutant tests" + type: boolean + default: false + stubs: + description: "Kotlin-name .pyi stubs" + type: boolean + default: true + +concurrency: + group: dev-ci + cancel-in-progress: false + +permissions: + contents: read + +jobs: + desktop: + if: ${{ inputs.desktop }} + uses: ./.github/workflows/desktop.yml + secrets: inherit + android: + if: ${{ inputs.android }} + uses: ./.github/workflows/android-native.yml + secrets: inherit + ios: + if: ${{ inputs.ios }} + uses: ./.github/workflows/ios.yml + secrets: inherit + wasm: + if: ${{ inputs.wasm }} + uses: ./.github/workflows/wasm.yml + secrets: inherit + typedpython: + if: ${{ inputs.typedpython }} + uses: ./.github/workflows/typedpython.yml + with: + race_mutant: ${{ inputs.race_mutant }} + secrets: inherit + stubs: + if: ${{ inputs.stubs }} + uses: ./.github/workflows/stubs.yml + permissions: + contents: write + secrets: inherit diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 0b43ff3e5..fab85eef2 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -1,16 +1,9 @@ name: Desktop CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: test: @@ -51,6 +44,84 @@ jobs: # Using --console=plain so logs are readable in GitHub Actions run: ./gradlew :python-multiplatform:desktopTest --console=plain + plugin-unit: + # The Gradle plugin's own unit tests (#403). Only its fixtures ran in CI before, so plugin + # unit tests were never executed (leader decision 2026-11-13). Linux: the tests use fake + # trees and need no Xcode or simulator. + name: Gradle plugin unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + cache: 'gradle' + + - name: Run the plugin's unit tests + # --rerun: never read an earlier result as this run's (AGENTS.md rule 8). + run: ./gradlew :python-multiplatform-gradle-plugin:test --rerun --console=plain + + - name: Count the tests + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import glob, xml.etree.ElementTree as ET + files = glob.glob("python-multiplatform-gradle-plugin/build/test-results/test/*.xml") + total = failed = skipped = 0 + for f in files: + suite = ET.parse(f).getroot() + total += int(suite.get("tests", 0)) + failed += int(suite.get("failures", 0)) + int(suite.get("errors", 0)) + skipped += int(suite.get("skipped", 0)) + print(f"Gradle plugin unit tests: {total} tests, {total - failed - skipped} passed, {failed} failed, {skipped} skipped ({len(files)} suites)") + if not files: + raise SystemExit("no plugin test results were written") + PY + cat "$GITHUB_STEP_SUMMARY" + + - name: Upload the test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gradle-plugin-unit-reports + path: | + python-multiplatform-gradle-plugin/build/reports/tests/test + python-multiplatform-gradle-plugin/build/test-results/test + + desktop-native-linux: + name: Desktop Kotlin/Native Tests (Linux x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + cache: 'gradle' + + - name: Cache Konan + uses: actions/cache@v4 + with: + path: ~/.konan + key: ${{ runner.os }}-konan-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-konan- + + - name: Run Linux x86_64 Kotlin/Native Tests + run: ./gradlew :python-multiplatform:linuxX64Test --rerun --console=plain + + # The Compose host's linuxX64 target (#386): compiled here, since only the fork publishes the + # Compose UI klibs for linuxX64. + - name: Compile the Compose host for Linux x86_64 Kotlin/Native + run: ./gradlew :python-multiplatform-compose:compileKotlinLinuxX64 --rerun --console=plain + # The dev agent's protocol on the other desktop OSes (#238). Its socket options differ per platform # (SO_EXCLUSIVEADDRUSE on Windows, SO_REUSEADDR on POSIX), and the port-takeover and restart tests are # what check them. Stdlib only: no JDK, no Gradle. @@ -151,26 +222,15 @@ jobs: python-multiplatform-compose/fixtures/notebook-e2e/build/reports/tests/desktopTest python-multiplatform-compose/fixtures/notebook-e2e/build/test-results/desktopTest - # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes `ui`, `ui-text` and - # `foundation` from compose-multiplatform-core-extended, which a runner reads from the fork's GitHub - # Packages registry; until the repository variable COMPOSE_EXTENDED_VERSION names a published - # development version (`1.11.1-ext.N-dev.`) the job is skipped. + # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes the fork's Compose + # (`org.thisisthepy.compose.*:1.11.1-ext.2`, `composeExtended.*` in `gradle.properties`) from + # Maven Central, builds the 20-scenario executable with Liberica NIK 25, and runs it with + # JAVA_HOME and PYTHONHOME removed. notebook-e2e-native-image: name: Notebook E2E in a native image (macOS) - if: ${{ vars.COMPOSE_EXTENDED_VERSION != '' }} runs-on: macos-latest - permissions: - contents: read - packages: read env: WHEEL_DIR: ${{ github.workspace }}/.tmp/pythonx-compose-dist - COMPOSE_EXTENDED_VERSION: ${{ vars.COMPOSE_EXTENDED_VERSION }} - COMPOSE_EXTENDED_GROUP_PREFIX: ${{ vars.COMPOSE_EXTENDED_GROUP_PREFIX || 'org.thisisthepy.compose' }} - COMPOSE_EXTENDED_REPOSITORY: ${{ vars.COMPOSE_EXTENDED_REPOSITORY || 'https://maven.pkg.github.com/thisisthepy/compose-multiplatform-core-extended' }} - # GitHub Packages asks for a token even for a public package. A read:packages PAT in the secret - # GH_PACKAGES_TOKEN is used when it exists; otherwise this repository's own token. - GITHUB_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GITHUB_TOKEN }} - TOKEN_SOURCE: ${{ secrets.GH_PACKAGES_TOKEN != '' && 'GH_PACKAGES_TOKEN' || 'GITHUB_TOKEN' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -183,19 +243,6 @@ jobs: path: .tmp/pythonx-compose fetch-depth: 1 - # Before any build: can this token read the fork's packages? One known POM, so a refusal is - # reported as what it is rather than as a dependency resolution error minutes later. - - name: Check access to the fork's packages - run: | - group_path="${COMPOSE_EXTENDED_GROUP_PREFIX//.//}/ui" - pom="$COMPOSE_EXTENDED_REPOSITORY/$group_path/ui-desktop/$COMPOSE_EXTENDED_VERSION/ui-desktop-$COMPOSE_EXTENDED_VERSION.pom" - status=$(curl -s -o /dev/null -w '%{http_code}' -u "${GITHUB_ACTOR}:${GITHUB_TOKEN}" "$pom") - echo "reading $pom with $TOKEN_SOURCE: HTTP $status" - if [ "$status" != "200" ]; then - echo "::error::The fork's packages could not be read with $TOKEN_SOURCE (HTTP $status at $pom). Add a personal access token (classic) with read:packages as the repository secret GH_PACKAGES_TOKEN, or check COMPOSE_EXTENDED_VERSION / COMPOSE_EXTENDED_REPOSITORY." - exit 1 - fi - - name: Setup uv uses: astral-sh/setup-uv@v6 @@ -220,13 +267,13 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside + # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` + # explicitly so Gradle's BellSoft JDK 25 toolchain resolution finds it. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" - -PcomposeExtended.version="$COMPOSE_EXTENDED_VERSION" - -PcomposeExtended.groupPrefix="$COMPOSE_EXTENDED_GROUP_PREFIX" - -PcomposeExtended.mavenLocal=false - -PcomposeExtended.repository="$COMPOSE_EXTENDED_REPOSITORY" + -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index f1287532b..ee7ee9fb6 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -1,16 +1,9 @@ name: iOS CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml new file mode 100644 index 000000000..10d22994d --- /dev/null +++ b/.github/workflows/maven-central.yml @@ -0,0 +1,137 @@ +name: Maven Central + +# Publishes python-multiplatform (library for every target, KSP processor, Gradle plugin and its +# plugin marker) to Maven Central through the Sonatype Central Portal, as ONE deployment. +# +# - A version tag (v-, the tag desktop-release.yml also reads) +# builds, signs with the release key and uploads. +# - A manual run is a dry run unless "dry_run" is turned off: it builds and signs with a key made in +# the job and thrown away, checks the bundle and keeps it as a workflow artifact. Nothing is +# uploaded. With dry_run off, pick the tag to publish in "tag". +# +# macOS is required: the Apple targets (iosArm64, iosSimulatorArm64, iosX64, macosArm64) only build +# there. Secrets read by a real upload, nothing else reads them: +# MAVEN_GPG_PRIVATE_KEY signing key, ASCII armoured (gpg --armor --export-secret-keys) +# MAVEN_GPG_PASSPHRASE its passphrase +# CENTRAL_TOKEN_USERNAME Central Portal user token, username half +# CENTRAL_TOKEN_PASSWORD Central Portal user token, password half +# The upload runs in the environment "maven-central", where required reviewers can be set. + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + dry_run: + description: 'Build and sign with a throwaway key, upload nothing' + type: boolean + default: true + tag: + description: 'Tag to publish when dry_run is off (e.g. v3.15.0-alpha02); ignored for a dry run' + required: false + default: '' + publishing_type: + description: 'USER_MANAGED waits in the Central Portal for someone to press Publish; AUTOMATIC publishes once validated' + type: choice + options: [USER_MANAGED, AUTOMATIC] + default: USER_MANAGED + +permissions: + contents: read + +concurrency: + group: maven-central-${{ github.event.inputs.tag || github.ref }} + cancel-in-progress: false + +jobs: + bundle: + name: Build and sign the bundle (macOS) + runs-on: macos-latest + timeout-minutes: 180 + env: + DRY_RUN: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }} + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ (github.event_name == 'push' || inputs.dry_run == false) && (github.event.inputs.tag || github.ref) || github.ref }} + # The tag's own gradle.properties decides the version, not the branch it was cut from. + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + + - name: Setup uv + # The KSP processor's typedpython-gate bundle is built with uv build. + uses: astral-sh/setup-uv@v6 + + - name: The tag names this build's version + if: env.DRY_RUN != 'true' + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + test "$TAG" = "v$version" || { echo "error: tag $TAG does not match version $version" >&2; exit 1; } + echo "VERSION=$version" >> "$GITHUB_ENV" + + - name: Build and sign the bundle (dry run, throwaway key) + if: env.DRY_RUN == 'true' + run: .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" --throwaway-key + + - name: Build and sign the bundle (release key) + if: env.DRY_RUN != 'true' + env: + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + set -euo pipefail + [[ -n "$MAVEN_GPG_PRIVATE_KEY" ]] || { echo "::error::the MAVEN_GPG_PRIVATE_KEY secret is not set"; exit 1; } + .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" + + - name: Keep the bundle + uses: actions/upload-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }}/central-bundle.zip + retention-days: 14 + + + upload: + name: Upload to the Central Portal + needs: bundle + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + # Required reviewers can be set on this environment, so even a pushed tag waits for a person + # before anything reaches Central. A dry run never gets here. + environment: maven-central + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + sparse-checkout: | + .github/scripts/release + gradle.properties + sparse-checkout-cone-mode: false + + - uses: actions/download-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }} + + - name: Upload + env: + CENTRAL_TOKEN_USERNAME: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + CENTRAL_TOKEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + TYPE: ${{ inputs.publishing_type || 'USER_MANAGED' }} + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + .github/scripts/release/central-upload.sh "$RUNNER_TEMP/central-bundle.zip" \ + "python-multiplatform $version" "$TYPE" diff --git a/.github/workflows/stubs.yml b/.github/workflows/stubs.yml index ed99ecd36..9cfb2879f 100644 --- a/.github/workflows/stubs.yml +++ b/.github/workflows/stubs.yml @@ -4,17 +4,16 @@ name: Kotlin-name stubs # but cannot run this repository's Gradle build. This job runs the stub task over the Compose version the # build resolves (`compose-libraries` in gradle/libs.versions.toml) and publishes the result. # -# - every push to develop and every pull request: the stubs are generated, type-checked against +# - every ci.yml run: the stubs are generated, type-checked against # .github/scripts/stubs/consumer.py with mypy (issue #31), and uploaded as the workflow artifact `kotlin-stubs`; # - every `v*` tag: the same stubs are also attached to the tag's GitHub release as `kotlin-stubs.zip`. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. + # Development runs come only through ci.yml, which is started by hand (owner, 2026-11-14: + # "개발 테스트 ci는 명시적으로 실행하기"). A release tag still attaches the stubs. push: - branches: [ "develop" ] tags: [ "v*" ] - workflow_dispatch: + workflow_call: permissions: contents: read diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml index 7c0cee003..b8b266d9b 100644 --- a/.github/workflows/typedpython.yml +++ b/.github/workflows/typedpython.yml @@ -3,17 +3,16 @@ name: TypedPython CI # The TypedPython gate and compiler are Python (python-multiplatform-ksp/src/{main,test}/python). # Gradle does not run them; this workflow does. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - paths: [ "python-multiplatform-ksp/**", "python-multiplatform-ksp/fixtures/app/**", ".github/workflows/typedpython.yml" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: + inputs: + race_mutant: + # The race_mutant tests show that a concurrency test catches a planted mutant; each depends + # on the race firing, so they run only when asked (leader decision 2026-11-13, #394). + description: "Also run the race_mutant tests (a missed race fails the step)" + type: boolean + default: false jobs: test: @@ -50,6 +49,10 @@ jobs: - name: Behaviour tests run: ${{ matrix.venv_python }} -m pytest -q -m "not measurement and not race_mutant" + - name: Race mutants (dispatch input race_mutant) + if: ${{ inputs.race_mutant }} + run: ${{ matrix.venv_python }} -m pytest -q -rA -m race_mutant + - name: Measurements (recorded, sanity bounds only) # A shared runner is not an idle machine (AGENTS.md §8): read ratios, not absolute times. run: ${{ matrix.venv_python }} -m pytest -q -s -m measurement diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 43320e8cd..bb0149b7b 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -13,16 +13,9 @@ name: WasmJS CI # `test` runs the real suite, and runs only when there is a runtime to run it against. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/.gitignore b/.gitignore index 19eb3c1c2..b3bee9b31 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,10 @@ google-services.json # Local Claude Code settings: per machine, not part of the repository. .claude/ .cache/ + +# TypedPython benchmark harness: uv venv, Cython/numba build output and numba caches +benchmarks/typedpython/.venv/ +benchmarks/typedpython/build/ +benchmarks/typedpython/target/ +benchmarks/typedpython/java-out/ +benchmarks/typedpython/tp-build/ diff --git a/README.md b/README.md index f126a862d..9e121c936 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ English | [한국어](docs/locale/README_ko.md) [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![Kotlin Multiplatform](https://img.shields.io/badge/Kotlin-Multiplatform-7F52FF.svg?logo=kotlin&logoColor=white)](https://kotlinlang.org/docs/multiplatform.html) -[![CPython](https://img.shields.io/badge/CPython-3.14-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) +[![CPython](https://img.shields.io/badge/CPython-3.15-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) [![Platforms](https://img.shields.io/badge/platforms-JVM%20%7C%20Android%20%7C%20iOS%20%7C%20Android%20native%20%7C%20wasm%20(exp.)-lightgrey.svg)](#-platforms) [Guide](https://thisisthepy.github.io/python-multiplatform/) · @@ -74,12 +74,16 @@ fun main() { val numbers = PyList.fromList(listOf(2L, 3L, 5L, 7L, 11L).map { PyInt.from(it) }) Python3.import("__main__").setAttr("kotlin_numbers", numbers) - val globals = Python3.import("__main__").dict - val result: PyObject = Python3.eval("sum(kotlin_numbers) * 2", 258 /* Py_eval_input */, globals, globals) + // One expression, evaluated in __main__. + val result: PyObject = Python3.eval("sum(kotlin_numbers) * 2") println("${result.Type.name}: $result") // int: 56 Python3.exec("print('hello from python')") // No close() anywhere: each wrapper releases its reference when it is collected. + + // Python buffers sys.stdout. finalize() flushes it and ends the interpreter (call it on the + // thread that called initialize()); Python3.flush() flushes without ending it. + Python3.finalize() } ``` @@ -198,7 +202,7 @@ Honest summary, see the [roadmap](docs/roadmap/ROADMAP.md) for what is left to d | Binding prebuilt jars, Compose from Python | ✅ desktop · ⏳ Android / iOS / wasm | | Binding Kotlin/Native klibs | 🟡 scanner only | | `.pyi` stub generation | 🟡 partial | -| Free-threaded CPython (3.14t, desktop only, opt-in `-PpythonFreeThreaded=true`) | 🟡 partial | +| Free-threaded CPython 3.15t (the only build on desktop since #158 and on Android / iOS since #328; wasm GIL until #175) | 🟡 desktop · 🟡 Android / iOS (compiled, not yet run on emulator or simulator) · ⏳ wasm | ## 🧱 The ecosystem diff --git a/benchmarks/typedpython/cython/binary_trees.pyx b/benchmarks/typedpython/cython/binary_trees.pyx new file mode 100644 index 000000000..1f25e4e66 --- /dev/null +++ b/benchmarks/typedpython/cython/binary_trees.pyx @@ -0,0 +1,46 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/binary_trees.py: one heap object per node, reference counted, freed by +# refcounting (not an arena). Semantics that differ from CPython: Node is a cdef class (no __dict__, +# attributes not visible from Python), C long counters, recursion uses the C stack (no +# RecursionError), no None checks on typed attributes. +import sys + + +cdef class Node: + cdef Node left + cdef Node right + + def __init__(self, Node left, Node right): + self.left = left + self.right = right + + +cdef Node make(int depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +cdef long check(Node node) noexcept: + cdef Node left = node.left + cdef Node right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +def main(): + cdef int n = int(sys.argv[1]) + cdef int min_depth = 4 + cdef int max_depth = max(min_depth + 2, n) + cdef int depth, i + cdef long iterations, chk, total = 0 + cdef long stretch = check(make(max_depth + 1)) + cdef Node long_lived = make(max_depth) + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for i in range(iterations): + chk += check(make(depth)) + total += chk + print(f"{stretch} {total} {check(long_lived)}") diff --git a/benchmarks/typedpython/cython/build.py b/benchmarks/typedpython/cython/build.py new file mode 100644 index 000000000..1ed61b65a --- /dev/null +++ b/benchmarks/typedpython/cython/build.py @@ -0,0 +1,46 @@ +"""Build the Cython columns. Run with the harness venv's python (see ../README in run.py's docstring). + + .venv/bin/python cython/build.py + +Writes /pure/.so (the unmodified py/*.py compiled in pure Python mode, annotations +as written) and /typed/_typed.so (cython/*.pyx). Not part of any timed run. +Flags: -O3 -ffp-contract=off (no fused multiply-add, so float results match CPython bit for bit). +""" +import shutil +import sys +from pathlib import Path + +from Cython.Build import cythonize +from setuptools import Extension, setup + +HERE = Path(__file__).resolve().parent +PY = HERE.parent / "py" +NAMES = {"nbody": "nbody", "spectral_norm": "spectral_norm", "fannkuch": "fannkuch", + "binary_trees": "binary_trees", "wordfreq": "wordfreq"} +CFLAGS = ["-O3", "-ffp-contract=off"] + + +def build(out: Path, kind: str) -> None: + work = out / f"src-{kind}" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + exts = [] + for name in NAMES: + if kind == "pure": + src = work / f"{name}.py" + shutil.copy(PY / f"{name}.py", src) # unmodified copy + mod = name + else: + mod = f"{name}_typed" + src = work / f"{mod}.pyx" + shutil.copy(HERE / f"{name}.pyx", src) + exts.append(Extension(mod, [str(src)], extra_compile_args=CFLAGS)) + ext_modules = cythonize(exts, language_level=3, quiet=True, build_dir=str(work / "c")) + setup(name=f"tp-bench-{kind}", ext_modules=ext_modules, script_name="setup.py", + script_args=["build_ext", "--build-lib", str(out / kind), "--build-temp", str(work / "obj"), "-q"]) + + +if __name__ == "__main__": + target = Path(sys.argv[1]).resolve() + for k in ("pure", "typed"): + build(target, k) diff --git a/benchmarks/typedpython/cython/fannkuch.pyx b/benchmarks/typedpython/cython/fannkuch.pyx new file mode 100644 index 000000000..9927c47fc --- /dev/null +++ b/benchmarks/typedpython/cython/fannkuch.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/fannkuch.py. Semantics that differ from CPython: C int arrays and C long +# counters (wrap instead of growing), no bounds checks, fixed capacity of 16 elements (n > 16 raises). +import sys + + +cdef (long, long) fannkuch(int n) noexcept: + cdef int perm1[16] + cdef int count[16] + cdef int perm[16] + cdef long max_flips = 0, checksum = 0, perm_count = 0, flips + cdef int r = n, i, k, lo, hi, t, p0 + for i in range(n): + perm1[i] = i + count[i] = 0 + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def main(): + cdef int n = int(sys.argv[1]) + if n > 16: + raise ValueError("fannkuch.pyx supports n <= 16") + cdef long checksum, max_flips + checksum, max_flips = fannkuch(n) + print(f"{checksum} {max_flips}") diff --git a/benchmarks/typedpython/cython/nbody.pyx b/benchmarks/typedpython/cython/nbody.pyx new file mode 100644 index 000000000..a12fd632a --- /dev/null +++ b/benchmarks/typedpython/cython/nbody.pyx @@ -0,0 +1,80 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/nbody.py. Semantics that differ from CPython: float division does not raise +# ZeroDivisionError (cdivision), no bounds checks (arrays are C arrays), loop counters are C long. +from libc.math cimport sqrt +import sys + +cdef double PI = 3.141592653589793 +cdef double SOLAR_MASS = 4 * PI * PI +cdef double DAYS_PER_YEAR = 365.24 + + +cdef void offset_momentum(double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef double px = 0.0, py = 0.0, pz = 0.0 + cdef int i + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +cdef void advance(double dt, long steps, double* x, double* y, double* z, + double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef long s + cdef int i, j + cdef double dx, dy, dz, d2, mag + for s in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +cdef double energy(double* x, double* y, double* z, double* vx, double* vy, double* vz, + double* mass) noexcept nogil: + cdef double e = 0.0, dx, dy, dz + cdef int i, j + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def main(): + cdef long n = int(sys.argv[1]) + cdef double[5] x = [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01] + cdef double[5] y = [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01] + cdef double[5] z = [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01] + cdef double[5] vx = [0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR] + cdef double[5] vy = [0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR] + cdef double[5] vz = [0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR] + cdef double[5] mass = [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS] + offset_momentum(vx, vy, vz, mass) + cdef double before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + cdef double after = energy(x, y, z, vx, vy, vz, mass) + print(f"{before:.9f} {after:.9f}") diff --git a/benchmarks/typedpython/cython/spectral_norm.pyx b/benchmarks/typedpython/cython/spectral_norm.pyx new file mode 100644 index 000000000..0b73cd8bc --- /dev/null +++ b/benchmarks/typedpython/cython/spectral_norm.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/spectral_norm.py. Semantics that differ from CPython: loop and index +# arithmetic is C long (wraps instead of growing; not reachable at these sizes), cdivision (no +# ZeroDivisionError), no bounds checks (malloc'd C arrays). +from libc.math cimport sqrt +from libc.stdlib cimport malloc, free +import sys + + +cdef inline double eval_a(long i, long j) noexcept nogil: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +cdef void mul_av(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +cdef void mul_atv(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +cdef void mul_atav(long n, double* v, double* out, double* tmp) noexcept nogil: + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +def main(): + cdef long n = int(sys.argv[1]) + cdef long i + cdef double* u = malloc(n * sizeof(double)) + cdef double* v = malloc(n * sizeof(double)) + cdef double* tmp = malloc(n * sizeof(double)) + cdef double vbv = 0.0, vv = 0.0 + try: + for i in range(n): + u[i] = 1.0 + v[i] = 0.0 + tmp[i] = 0.0 + for i in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + finally: + free(u) + free(v) + free(tmp) + print(f"{sqrt(vbv / vv):.9f}") diff --git a/benchmarks/typedpython/cython/wordfreq.pyx b/benchmarks/typedpython/cython/wordfreq.pyx new file mode 100644 index 000000000..2b1734a91 --- /dev/null +++ b/benchmarks/typedpython/cython/wordfreq.pyx @@ -0,0 +1,54 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/wordfreq.py: the counts stay a Python dict[str, int] (the algorithm's data +# structure), only the arithmetic is typed and words are built in a C buffer. Semantics that differ +# from CPython: C long arithmetic (wraps; the LCG product stays below 2**47 and the weighted sum +# below 2**63 at benchmark size), cdivision. +import sys + +cdef long VOCAB = 50000 +cdef long MOD = 2147483647 + + +cdef inline long next_state(long s) noexcept nogil: + return (s * 48271) % MOD + + +cdef str word_of(long word_id): + cdef char buf[32] + cdef int pos = 32 + cdef long x = word_id + 17576 + while x > 0: + pos -= 1 + buf[pos] = (97 + x % 26) + x //= 26 + return (buf[pos:32]).decode("ascii") + + +def main(): + cdef long n = int(sys.argv[1]) + cdef dict counts = {} + cdef long s = 12345, a, b, i, c, weighted = 0, best_count + cdef str word, w, best_word + cdef list taken = [] + cdef list parts = [] + for i in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + counts[word] = counts.get(word, 0) + 1 + for w, c in counts.items(): + weighted += c * c + for i in range(5): + best_word = "" + best_count = -1 + for w, c in counts.items(): + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + parts.append(f"{best_word}:{best_count}") + print(f"{len(counts)} {weighted} {','.join(parts)}") diff --git a/benchmarks/typedpython/numba/binary_trees.py b/benchmarks/typedpython/numba/binary_trees.py new file mode 100644 index 000000000..35a4b7fcc --- /dev/null +++ b/benchmarks/typedpython/numba/binary_trees.py @@ -0,0 +1,72 @@ +"""numba version of py/binary_trees.py, using numba.experimental.jitclass. + +Node is a jitclass with a deferred (self-referential) Optional type, the closest numba has to a +heap object per node; allocation and release go through numba's NRT reference counting, like the +CPython program. (An index-based tree in preallocated arrays would be faster but changes the +allocation pattern into an arena, so it is not used.) + +Semantics that differ from CPython: counters are int64, recursion uses the native stack (no +RecursionError). The module must be registered in sys.modules before it runs (run.py does). +""" +from numba import deferred_type, int64, njit, optional +from numba.core.types import UniTuple +from numba.experimental import jitclass + +node_type = deferred_type() + + +@jitclass([("left", optional(node_type)), ("right", optional(node_type))]) +class Node: + def __init__(self, left, right): + self.left = left + self.right = right + + +NodeT = Node.class_type.instance_type +node_type.define(NodeT) + +# cache=True is not usable here: numba raises AttributeError: 'Optional' object has no attribute +# 'type' while hashing the recursive jitclass type for the cache index. So this benchmark has no +# cached run; every process compiles (at import, see below). +OPTS = dict(cache=False) + + +# Explicit signatures: numba cannot infer the return type of the self-recursive make() through +# the jitclass's deferred Optional type (AttributeError: 'Optional' object has no attribute +# 'type'). The consequence is that these two functions compile when this module is imported, not +# on the first call; run.py counts the import time as compile time. +@njit(NodeT(int64), **OPTS) +def make(depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +@njit(int64(optional(NodeT)), **OPTS) +def check(node): + left = node.left + right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +@njit(UniTuple(int64, 3)(int64), **OPTS) +def kernel(n): + min_depth = 4 + max_depth = max(min_depth + 2, n) + stretch = check(make(max_depth + 1)) + long_lived = make(max_depth) + total = 0 + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for _ in range(iterations): + chk += check(make(depth)) + total += chk + return stretch, total, check(long_lived) + + +def run(n: int) -> str: + stretch, total, long_lived = kernel(n) + return f"{stretch} {total} {long_lived}" diff --git a/benchmarks/typedpython/numba/fannkuch.py b/benchmarks/typedpython/numba/fannkuch.py new file mode 100644 index 000000000..5f21f092e --- /dev/null +++ b/benchmarks/typedpython/numba/fannkuch.py @@ -0,0 +1,62 @@ +"""numba @njit version of py/fannkuch.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: int64 numpy arrays and int64 counters (wrap instead of +growing), no bounds checks. +""" +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def fannkuch(n): + perm1 = np.arange(n) + count = np.zeros(n, dtype=np.int64) + perm = np.zeros(n, dtype=np.int64) + max_flips = 0 + checksum = 0 + perm_count = 0 + r = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def run(n: int) -> str: + checksum, max_flips = fannkuch(n) + return f"{checksum} {max_flips}" diff --git a/benchmarks/typedpython/numba/nbody.py b/benchmarks/typedpython/numba/nbody.py new file mode 100644 index 000000000..74d8b8bee --- /dev/null +++ b/benchmarks/typedpython/numba/nbody.py @@ -0,0 +1,82 @@ +"""numba @njit version of py/nbody.py. Loaded by run.py; `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays instead of lists; error_model="numpy" +(float division by zero gives inf instead of raising ZeroDivisionError); no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +PI = 3.141592653589793 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def offset_momentum(vx, vy, vz, mass): + px = 0.0 + py = 0.0 + pz = 0.0 + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +@njit(**OPTS) +def advance(dt, steps, x, y, z, vx, vy, vz, mass): + for _ in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * math.sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +@njit(**OPTS) +def energy(x, y, z, vx, vy, vz, mass): + e = 0.0 + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def run(n: int) -> str: + x = np.array([0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01]) + y = np.array([0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01]) + z = np.array([0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01]) + vx = np.array([0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR]) + vy = np.array([0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR]) + vz = np.array([0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR]) + mass = np.array([SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS]) + offset_momentum(vx, vy, vz, mass) + before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + after = energy(x, y, z, vx, vy, vz, mass) + return f"{before:.9f} {after:.9f}" diff --git a/benchmarks/typedpython/numba/spectral_norm.py b/benchmarks/typedpython/numba/spectral_norm.py new file mode 100644 index 000000000..bfa99bae5 --- /dev/null +++ b/benchmarks/typedpython/numba/spectral_norm.py @@ -0,0 +1,61 @@ +"""numba @njit version of py/spectral_norm.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays, int64 index arithmetic (wraps), +error_model="numpy" (no ZeroDivisionError), no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def eval_a(i, j): + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +@njit(**OPTS) +def mul_av(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atv(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atav(n, v, out, tmp): + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +@njit(**OPTS) +def kernel(n): + u = np.ones(n) + v = np.zeros(n) + tmp = np.zeros(n) + for _ in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + vbv = 0.0 + vv = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + return vbv, vv + + +def run(n: int) -> str: + vbv, vv = kernel(n) + return f"{math.sqrt(vbv / vv):.9f}" diff --git a/benchmarks/typedpython/numba/wordfreq.py b/benchmarks/typedpython/numba/wordfreq.py new file mode 100644 index 000000000..5bc8599fb --- /dev/null +++ b/benchmarks/typedpython/numba/wordfreq.py @@ -0,0 +1,69 @@ +"""numba @njit version of py/wordfreq.py: numba.typed.Dict[unicode_type, int64]. + +Semantics that differ from CPython: int64 arithmetic (wraps), words are numba unicode strings, +the counts dict is a numba typed.Dict (hash order differs, but the result does not depend on +iteration order because ties are broken by string comparison). +""" +import numpy as np +from numba import njit, types +from numba.typed import Dict, List + +VOCAB = 50000 +MOD = 2147483647 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def next_state(s): + return (s * 48271) % MOD + + +@njit(**OPTS) +def word_of(word_id): + x = word_id + 17576 + w = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w + + +@njit(**OPTS) +def kernel(n): + counts = Dict.empty(key_type=types.unicode_type, value_type=types.int64) + s = 12345 + for _ in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + if word in counts: + counts[word] += 1 + else: + counts[word] = 1 + weighted = 0 + for w in counts: + c = counts[w] + weighted += c * c + taken = List.empty_list(types.unicode_type) + best_counts = np.zeros(5, dtype=np.int64) + for k in range(5): + best_word = "" + best_count = -1 + for w in counts: + c = counts[w] + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + best_counts[k] = best_count + return len(counts), weighted, taken, best_counts + + +def run(n: int) -> str: + size, weighted, taken, best_counts = kernel(n) + parts = [f"{w}:{c}" for w, c in zip(taken, best_counts)] + return f"{size} {weighted} {','.join(parts)}" diff --git a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py index a0cd59c56..53a917186 100644 --- a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py +++ b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import array +@compiled def dot_arr(a: array.array[float], b: array.array[float]) -> float: s: float = 0.0 for i in range(len(a)): @@ -9,11 +9,13 @@ def dot_arr(a: array.array[float], b: array.array[float]) -> float: return s +@compiled def scale_arr(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k +@compiled def dot_list(a: list[float], b: list[float]) -> float: s: float = 0.0 for i in range(len(a)): diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 104ad572d..cf8aebf5a 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -10,17 +10,40 @@ The "TypedPython @compiled" column runs only with `--typedpython ` (the directory that holds the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with -the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is +`@compiled` above every top-level def and class, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. +Cython and numba columns (issue: compare TypedPython fairly against them). They need a harness venv +that holds Cython, numba and a C compiler (Xcode command line tools), created ONLY with uv: + + uv venv --python 3.14 benchmarks/typedpython/.venv + uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba + +`--tools-python` (default benchmarks/typedpython/.venv/bin/python) names that interpreter; without it +the three columns report n/a. Columns, each in the form most favourable to it: + "Cython (pure)" py/*.py compiled unmodified by cythonize (annotations as written), same CPython; + "Cython (typed)" cython/*.pyx: cdef types, C arrays, boundscheck/wraparound off, cdef class Node; + "numba" numba/*.py with @njit(cache=True) (binary-trees: jitclass, no cache). The time + reported is the STEADY-STATE call at benchmark size, inside the process, after a + warm-up call that triggered the JIT. The JIT cost is the separate record + "numba compile" (first call: cold cache vs cached; numba import excluded). +Cython is built by cython/build.py before any timing (build/cython/, git-ignored). numba runs in a +fresh process with its own NUMBA_CACHE_DIR (build/numba-cache/, wiped per cold run). +"CPython (tools python)" is the same program under the tools interpreter, and is the baseline for the +Cython/numba ratios in `interleaved` (so those pairs use one interpreter); it reads "same as CPython" +when --python and --tools-python are the same version. Per-column tool version, flags and a semantics +note are written to the JSON ("tools") and printed after the table. + binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The speedup is reported against that CPython version; `CPython (no __slots__)` runs the same program with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. """ import argparse +import ast import datetime import json import os +import shutil import socket import statistics import subprocess @@ -45,7 +68,31 @@ "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), } NOSLOTS = "CPython (no __slots__)" -COLUMNS = ["CPython", NOSLOTS, "Node", "Java", "Rust", "TypedPython @compiled"] +CPY_TOOLS = "CPython (tools python)" +CY_PURE = "Cython (pure)" +CY_TYPED = "Cython (typed)" +NUMBA = "numba" +TOOL_COLS = [CY_PURE, CY_TYPED, NUMBA] +COLUMNS = ["CPython", NOSLOTS, CPY_TOOLS, "Node", "Java", "Rust", "TypedPython @compiled", *TOOL_COLS] +TOOLS_PY_DEFAULT = ROOT / ".venv" / "bin" / "python" +CYTHON_OUT = ROOT / "build" / "cython" +NUMBA_CACHE = ROOT / "build" / "numba-cache" +# tiny sizes for numba's warm-up (first) call: the types are the same at any size, so the first +# call is almost all compile time +NUMBA_WARM = {"nbody": "10", "spectral-norm": "4", "fannkuch-redux": "4", "binary-trees": "1", + "wordfreq": "10"} +CYTHON_FLAGS = "cython/build.py: cythonize(language_level=3), cc -O3 -ffp-contract=off (no fused multiply-add)" +SEMANTICS = { + CY_PURE: "py/*.py unmodified; Cython honours the existing annotations (float is a C double, int stays " + "a Python int, list stays a list), so semantics are CPython's apart from float arithmetic on typed locals", + CY_TYPED: "C semantics where declared: C long/int arithmetic wraps instead of growing, cdivision (no " + "ZeroDivisionError), boundscheck/wraparound off on C arrays, cdef class Node (no __dict__), " + "C-stack recursion (no RecursionError); wordfreq keeps a Python dict", + NUMBA: "fixed-width int64/float64 (ints wrap, no Python big ints, no OverflowError), error_model=numpy " + "(no ZeroDivisionError), no bounds checks, numpy arrays for lists, typed.Dict/List for dict/list " + "(wordfreq), jitclass for the tree (binary-trees); reported time is steady state, JIT is separate", +} + SLOTS_LINE = ' __slots__ = ("left", "right")\n' NOSLOTS_BENCHMARKS = {"binary-trees"} NOT_BUILT = "n/a (not built)" @@ -87,6 +134,17 @@ def build() -> None: """ +def with_compiled(source: str) -> str: + """`source` with `@compiled` above every top-level def and class (above their other decorators).""" + tree = ast.parse(source) + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[str]] | str]: """Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason.""" built: dict[str, tuple[str, list[str]] | str] = {} @@ -95,7 +153,7 @@ def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[st work = TP_BUILD / name work.mkdir(parents=True, exist_ok=True) source = work / py - source.write_text("# typedpython: compiled\n" + (ROOT / "py" / py).read_text()) + source.write_text(with_compiled((ROOT / "py" / py).read_text())) result = subprocess.run([tp_python, "-c", COMPILE_SNIPPET, str(source), str(work / "out")], capture_output=True, text=True, env=env) if result.returncode != 0: @@ -118,8 +176,64 @@ def noslots_source(name: str) -> Path: return out +TOOLS_PROBE = """ +import json, sys +out = {"python": sys.version.split()[0]} +for mod in ("Cython", "numba", "llvmlite", "numpy", "setuptools"): + try: + out[mod] = __import__(mod).__version__ + except Exception as e: + out[mod] = f"missing ({type(e).__name__}: {e})" +print(json.dumps(out)) +""" + + +def _cython_up_to_date() -> bool: + sources = [*(ROOT / "py").glob("*.py"), *(ROOT / "cython").glob("*"), ] + newest = max(p.stat().st_mtime for p in sources) + for name, (py, *_r) in BENCHMARKS.items(): + stem = Path(py).stem + for kind, mod in (("pure", stem), ("typed", stem + "_typed")): + found = list((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if not found or found[0].stat().st_mtime < newest: + return False + return True + + +def build_tools(tools_python: str | None, python: str) -> dict[str, object]: + """Probe the tools interpreter and cythonize + C-compile the Cython columns (never in a timed run). + Returns {"python", "versions", "na": {column: reason}, "same_python"}.""" + tools: dict[str, object] = {"python": None, "versions": {}, "na": {}, "same_python": True} + na: dict[str, str] = tools["na"] # type: ignore[assignment] + if not tools_python or not Path(tools_python).exists(): + why = ("n/a (no harness venv; create it with: uv venv --python 3.14 benchmarks/typedpython/.venv && " + "uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba)") + return {**tools, "na": {c: why for c in TOOL_COLS}} + probe = subprocess.run([tools_python, "-c", TOOLS_PROBE], capture_output=True, text=True) + if probe.returncode != 0: + return {**tools, "na": {c: f"n/a (tools python failed: {probe.stderr.strip()[-200:]})" for c in TOOL_COLS}} + versions = json.loads(probe.stdout.strip().splitlines()[-1]) + tools.update(python=tools_python, versions=versions) + base = subprocess.run([python, "-c", "import sys; print(sys.version.split()[0])"], + capture_output=True, text=True).stdout.strip() + tools["same_python"] = base == versions["python"] + tools["baseline_python"] = base + if "missing" in versions["Cython"] or "missing" in versions["setuptools"]: + na[CY_PURE] = na[CY_TYPED] = f"n/a (Cython {versions['Cython']}; setuptools {versions['setuptools']})" + elif not _cython_up_to_date(): + build = subprocess.run([tools_python, str(ROOT / "cython" / "build.py"), str(CYTHON_OUT)], + capture_output=True, text=True) + if build.returncode != 0: + reason = f"n/a (cython build failed: {' | '.join(build.stderr.strip().splitlines()[-3:])})" + na[CY_PURE] = na[CY_TYPED] = reason + if "missing" in versions["numba"] or "missing" in versions["numpy"]: + na[NUMBA] = f"n/a (numba {versions['numba']}; numpy {versions['numpy']})" + return tools + + def commands(name: str, size: str, python: str, java_source: bool, - compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: + compiled: dict[str, tuple[str, list[str]] | str] | None = None, + tools: dict[str, object] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ else ["java", "-cp", str(JAVA_OUT), cls, size] @@ -134,6 +248,25 @@ def commands(name: str, size: str, python: str, java_source: bool, entry = (compiled or {}).get(name) if isinstance(entry, tuple): cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] + if tools and tools.get("python"): + tp = str(tools["python"]) + na: dict[str, str] = tools["na"] # type: ignore[assignment] + stem = Path(py).stem + cmds[CPY_TOOLS] = Cmd([tp, str(ROOT / "py" / py), size]) + for col, kind, mod in ((CY_PURE, "pure", stem), (CY_TYPED, "typed", stem + "_typed")): + so = sorted((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if col not in na and so: + cmd = Cmd([tp, "-c", RUN_SNIPPET, mod, str(so[0]), size]) + cmd.soft = True + cmds[col] = cmd + if NUMBA not in na: + cmd = Cmd([tp, "-c", NUMBA_SNIPPET, str(ROOT / "numba" / py), stem + "_numba", + NUMBA_WARM[name], size]) + cmd.numba = True + cmd.soft = True + cmd.cache_dir = NUMBA_CACHE / name + cmd.env = {"NUMBA_CACHE_DIR": str(cmd.cache_dir)} + cmds[NUMBA] = cmd return cmds @@ -145,34 +278,166 @@ def source_commit() -> str: return result.stdout.strip() + ("+dirty" if dirty else "") if result.returncode == 0 else "unknown" -def run_once(cmd: list[str]) -> tuple[float, str]: +class RunFailed(Exception): + """A Cython/numba column failed to run; reported as n/a with the concrete error.""" + + +class Cmd(list): + """argv plus, for the numba column, an environment, a cache directory and the last timing.""" + env: dict[str, str] | None = None + numba = False + cache_dir: Path | None = None + cold = True + last: dict[str, float] | None = None + soft = False + + +NUMBA_SNIPPET = """ +import importlib.util, json, sys, time +import numba, numpy # imported first: import time is not compile time +path, name, warm, size = sys.argv[1:5] +t0 = time.perf_counter() +spec = importlib.util.spec_from_file_location(name, path) +m = importlib.util.module_from_spec(spec) +sys.modules[name] = m # jitclass resolves its module through sys.modules +spec.loader.exec_module(m) # binary-trees (explicit signatures) compiles here +m.run(int(warm)) # first call: JIT compile (or cache load) at a tiny size +t1 = time.perf_counter() +out = m.run(int(size)) # steady state: already compiled +t2 = time.perf_counter() +print(out) +print("#timing " + json.dumps({"first_call_s": t1 - t0, "steady_s": t2 - t1})) +""" + + +def run_once(cmd: list[str], soft: bool = False) -> tuple[float, str]: + """Wall time and stdout. A numba Cmd returns its steady-state seconds instead (see NUMBA_SNIPPET).""" + cache_dir = getattr(cmd, "cache_dir", None) + if cache_dir is not None and getattr(cmd, "cold", True): + shutil.rmtree(cache_dir, ignore_errors=True) + env = dict(os.environ, **cmd.env) if getattr(cmd, "env", None) else None # type: ignore[attr-defined] start = time.perf_counter() - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, env=env) elapsed = time.perf_counter() - start if result.returncode != 0: + if soft or getattr(cmd, "soft", False): + last = (result.stderr.strip().splitlines() or ["no stderr"])[-1] + raise RunFailed(f"exit {result.returncode}: {last}") sys.stderr.write(f"failed ({result.returncode}): {' '.join(cmd)}\n{result.stderr}") sys.exit(1) - return elapsed, result.stdout.strip() + out = result.stdout.strip() + if getattr(cmd, "numba", False): + lines = out.splitlines() + timing = json.loads(lines[-1][len("#timing "):]) + cmd.last = dict(timing, wall_s=elapsed) # type: ignore[attr-defined] + out = "\n".join(lines[:-1]).strip() + elapsed = timing["steady_s"] + return elapsed, out + + +def tools_for(args: argparse.Namespace) -> dict[str, object]: + return build_tools(args.tools_python, args.python) + + +def tools_report(tools: dict[str, object]) -> dict[str, object]: + """Per-column tool version, flags and semantics note, for the printout and the JSON.""" + v: dict[str, str] = tools.get("versions") or {} # type: ignore[assignment] + na: dict[str, str] = tools["na"] # type: ignore[assignment] + py = f"CPython {v.get('python', '?')}" + report: dict[str, object] = { + "tools_python": {"interpreter": py, "baseline_python": tools.get("baseline_python"), + "same_interpreter_as_CPython_column": tools.get("same_python")}, + CY_PURE: {"tool": f"Cython {v.get('Cython')} on {py}", "flags": CYTHON_FLAGS, + "semantics": SEMANTICS[CY_PURE]}, + CY_TYPED: {"tool": f"Cython {v.get('Cython')} on {py}", + "flags": CYTHON_FLAGS + "; # cython: boundscheck=False, wraparound=False, cdivision=True, " + "initializedcheck=False in each .pyx", "semantics": SEMANTICS[CY_TYPED]}, + NUMBA: {"tool": f"numba {v.get('numba')}, llvmlite {v.get('llvmlite')}, numpy {v.get('numpy')} on {py}", + "flags": "@njit(cache=True, error_model='numpy'); binary-trees: jitclass + explicit " + "signatures, cache=False; NUMBA_CACHE_DIR per benchmark", + "semantics": SEMANTICS[NUMBA]}, + } + for col in TOOL_COLS: + if col in na: + report[col]["unavailable"] = na[col] # type: ignore[index] + return report + + +def print_tools_report(report: dict[str, object]) -> None: + print("\ntools:") + for key, val in report.items(): + if key == "tools_python": + print(f" tools python: {val['interpreter']}; baseline CPython column: {val['baseline_python']}; " # type: ignore[index] + f"same interpreter: {val['same_interpreter_as_CPython_column']}") # type: ignore[index] + else: + print(f" {key}: {val['tool']}\n flags: {val['flags']}\n semantics: {val['semantics']}") # type: ignore[index] + + +def run_column(col: str, cmd: list[str]) -> tuple[float, str] | str: + """run_once, but a Cython/numba failure becomes the n/a string instead of ending the run.""" + try: + return run_once(cmd) + except RunFailed as e: + return f"n/a ({e})" def verify(args: argparse.Namespace) -> int: build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) bad = 0 for name, spec in BENCHMARKS.items(): - outputs = {col: run_once(cmd)[1] - for col, cmd in commands(name, spec[4], args.python, True, compiled).items()} + outputs: dict[str, str] = {} + notes: dict[str, str] = dict(tools["na"]) # type: ignore[arg-type] + for col, cmd in commands(name, spec[4], args.python, True, compiled, tools).items(): + res = run_column(col, cmd) + if isinstance(res, str): + notes[col] = res + else: + outputs[col] = res[1] distinct = set(outputs.values()) status = "OK " if len(distinct) == 1 else "FAIL" print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") + print(f" columns: {', '.join(outputs)}") + for col, why in notes.items(): + print(f" {col}: {why}") if len(distinct) != 1: bad += 1 for col, out in outputs.items(): print(f" {col}: {out}") print("all outputs agree" if bad == 0 else f"{bad} benchmark(s) disagree") + print_tools_report(tools_report(tools)) return 1 if bad else 0 +def measure_numba(cmd: Cmd, repeat: int) -> tuple[dict[str, object], dict[str, object]]: + """Cold-cache runs (each wipes the cache dir, so each compiles) give the steady-state time and the + cold compile time; one more run with the cache left in place gives the cached first-call time.""" + steady, outs, cold_first, walls = [], set(), [], [] + for _ in range(repeat): + t, out = run_once(cmd) + steady.append(t) + outs.add(out) + cold_first.append(cmd.last["first_call_s"]) # type: ignore[index] + walls.append(cmd.last["wall_s"]) # type: ignore[index] + cache_files = len([p for p in cmd.cache_dir.rglob("*") if p.is_file()]) if cmd.cache_dir else 0 + cmd.cold = False + try: + t_c, out_c = run_once(cmd) + outs.add(out_c) + cached_first = cmd.last["first_call_s"] # type: ignore[index] + cached_steady = t_c + finally: + cmd.cold = True + column = {"median_s": statistics.median(steady), "runs_s": steady, "output": sorted(outs), + "process_wall_s_runs": walls, "note": "steady-state call at benchmark size, JIT excluded"} + compile_rec = {"cold_first_call_s_median": statistics.median(cold_first), "cold_first_call_s_runs": cold_first, + "cached_first_call_s": cached_first, "cached_steady_s": cached_steady, + "cache_files_written": cache_files, "cache_used": cache_files > 0, + "note": "first call = module import + first run() at a tiny size; numba import excluded"} + return column, compile_rec + + def measure(args: argparse.Namespace) -> int: load = os.getloadavg() print(f"load average: {load[0]:.2f} {load[1]:.2f} {load[2]:.2f}") @@ -182,6 +447,8 @@ def measure(args: argparse.Namespace) -> int: return 2 build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) + report = tools_report(tools) results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): size = args.size.get(name, spec[5]) @@ -189,15 +456,29 @@ def measure(args: argparse.Namespace) -> int: entry = (compiled or {}).get(name, NOT_BUILT) if isinstance(entry, tuple): row["compiled_functions"] = entry[1] - for col, cmd in commands(name, size, args.python, False, compiled).items(): - runs = [run_once(cmd) for _ in range(args.repeat)] - outs = {out for _, out in runs} - row[col] = {"median_s": statistics.median(t for t, _ in runs), - "runs_s": [t for t, _ in runs], "output": sorted(outs)} + for col, cmd in commands(name, size, args.python, False, compiled, tools).items(): + if col == CPY_TOOLS and tools.get("same_python"): + continue + try: + if col == NUMBA: + row[col], row["numba compile"] = measure_numba(cmd, args.repeat) # type: ignore[arg-type] + outs = set(row[col]["output"]) # type: ignore[index] + else: + runs = [run_once(cmd) for _ in range(args.repeat)] + outs = {out for _, out in runs} + row[col] = {"median_s": statistics.median(t for t, _ in runs), + "runs_s": [t for t, _ in runs], "output": sorted(outs)} + except RunFailed as e: + row[col] = f"n/a ({e})" + continue if len(outs) != 1: print(f"warning: {name}/{col} produced differing outputs across runs") if not isinstance(entry, tuple): row["TypedPython @compiled"] = entry + for col, why in tools["na"].items(): # type: ignore[union-attr] + row[col] = why + if tools.get("same_python"): + row[CPY_TOOLS] = "same as CPython" measured = [c for c in COLUMNS if isinstance(row.get(c), dict)] outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index] row["outputs_agree"] = len(outs_all) == 1 @@ -214,6 +495,13 @@ def measure(args: argparse.Namespace) -> int: flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") + print("numba column = steady-state call inside the process (JIT excluded); its compile cost:") + for name, row in results.items(): + rec = row.get("numba compile") + if isinstance(rec, dict): + print(f" {name}: cold first call {rec['cold_first_call_s_median']:.3f} s, cached first call " + f"{rec['cached_first_call_s']:.3f} s, cache files written {rec['cache_files_written']}") + print_tools_report(report) if args.write: out_dir = ROOT / "results" @@ -223,7 +511,7 @@ def measure(args: argparse.Namespace) -> int: "commit": source_commit(), "host": socket.gethostname(), "load_average": load, "repeat": args.repeat, "python": subprocess.run([args.python, "--version"], capture_output=True, text=True).stdout.strip(), - "results": results} + "tools": report, "results": results} path.write_text(json.dumps(payload, indent=2) + "\n") print(f"wrote {path}") return 0 @@ -233,46 +521,48 @@ def interleaved(args: argparse.Namespace) -> int: """CPython vs TypedPython @compiled, alternated A/B in the same time window (leader decision, 2026-10-03): when the machine is never quiet, load hits both sides of each pair alike, so the per-pair ratio is stable even though absolute times are not. Reports the median ratio, never - absolute times as the result, with the load average recorded at every pair.""" - if not args.typedpython: - print("interleaved needs --typedpython") + absolute times as the result, with the load average recorded at every pair. + + The Cython and numba columns are paired the same way against the CPython run under the tools + interpreter (`CPython (tools python)`), each pair alternating which side runs first. For numba the + ratio uses its in-process steady-state time (JIT excluded, process start included for CPython, a + small bias in numba's favour at these sizes); its first-call (compile) time is recorded per pair.""" + tools = tools_for(args) + if not args.typedpython and not tools.get("python"): + print("interleaved needs --typedpython or a tools python") return 2 - compiled = build_compiled(Path(args.typedpython), args.typedpython_python) + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else {} results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): entry = compiled.get(name) - if not isinstance(entry, tuple): - results[name] = {"skipped": entry} - continue size = args.size.get(name, spec[5]) - cmds = commands(name, size, args.python, False, compiled) - a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] - c_cmd = cmds.get(NOSLOTS) - pairs = [] - for i in range(args.pairs): - order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first - load = os.getloadavg() - (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) - t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) - if o_a != o_b: - print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") - return 1 - pair = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} - if c_cmd is not None: - t_c, o_c = run_once(c_cmd) - if o_c != o_a: - print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") - return 1 - pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) - pairs.append(pair) - ratios = [p["ratio"] for p in pairs] - results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, - "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), - "max_ratio": max(ratios)} - if c_cmd is not None: - results[name]["median_ratio_vs_noslots"] = statistics.median( - p["ratio_vs_noslots"] for p in pairs) + cmds = commands(name, size, args.python, False, compiled, tools) + r: dict[str, object] = {"size": size} + if args.typedpython: + if isinstance(entry, tuple): + r.update(measure_pair(name, cmds["CPython"], cmds["TypedPython @compiled"], cmds.get(NOSLOTS), + args.pairs, "compiled")) + r["compiled_functions"] = entry[1] + else: + r["skipped"] = entry + extras: dict[str, object] = {} + base = cmds.get(CPY_TOOLS) + for col in TOOL_COLS: + if col in tools["na"]: # type: ignore[operator] + extras[col] = tools["na"][col] # type: ignore[index] + elif col not in cmds or base is None: + extras[col] = "n/a (not built)" + else: + try: + extras[col] = measure_pair(name, base, cmds[col], None, args.pairs, col) + except RunFailed as e: + extras[col] = f"n/a ({e})" + r["tools_pairs"] = extras + results[name] = r print(f"measured {name}", file=sys.stderr) + if any(isinstance(v, dict) and v.get("outputs_differ") for v in [r, *extras.values()]): + print(f"{name}: OUTPUTS DIFFER, see the JSON/table") + return 1 print() print("| benchmark | size | speedup vs CPython (median of pairs) | range | load during pairs (1-min) |") @@ -280,11 +570,27 @@ def interleaved(args: argparse.Namespace) -> int: for name, r in results.items(): if "skipped" in r: print(f"| {name} | - | {r['skipped']} | | |") - continue - loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] - print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}–{r['max_ratio']:.1f}x " - f"| {min(loads):.1f}–{max(loads):.1f} |") - print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + elif "median_ratio" in r: + loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] + print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}\u2013{r['max_ratio']:.1f}x " + f"| {min(loads):.1f}\u2013{max(loads):.1f} |") + if args.typedpython: + print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + print("\n| benchmark | column | speedup vs CPython (tools python), median of pairs | range | extra |") + print("|---|---|---|---|---|") + for name, r in results.items(): + for col, p in r["tools_pairs"].items(): # type: ignore[union-attr] + if isinstance(p, dict): + extra = "" + if col == NUMBA: + extra = (f"first call (compile) median {statistics.median(q['first_call_s'] for q in p['pairs']):.3f} s; " + f"process wall median {statistics.median(q['wall_s'] for q in p['pairs']):.3f} s") + print(f"| {name} | {col} | {p['median_ratio']:.1f}x | {p['min_ratio']:.1f}\u2013{p['max_ratio']:.1f}x | {extra} |") + else: + print(f"| {name} | {col} | {p} | | |") + print(f"\n{args.pairs} pairs per column, order alternated; ratio = CPython (tools python) time / column time.") + report = tools_report(tools) + print_tools_report(report) if args.write: out_dir = ROOT / "results" out_dir.mkdir(exist_ok=True) @@ -292,11 +598,42 @@ def interleaved(args: argparse.Namespace) -> int: path.write_text(json.dumps({"date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": source_commit(), "host": socket.gethostname(), "method": "interleaved A/B", - "pairs": args.pairs, "results": results}, indent=2) + "\n") + "pairs": args.pairs, "tools": report, "results": results}, indent=2) + "\n") print(f"wrote {path}") return 0 +def measure_pair(name: str, a_cmd: list[str], b_cmd: list[str], c_cmd: list[str] | None, n_pairs: int, + label: str) -> dict[str, object]: + """n_pairs A/B pairs (A = CPython, B = the column); which runs first alternates every pair.""" + pairs = [] + for i in range(n_pairs): + order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first + load = os.getloadavg() + (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) + t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) + if o_a != o_b: + print(f"{name}/{label}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") + return {"outputs_differ": True} + pair: dict[str, object] = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} + last = getattr(b_cmd, "last", None) + if getattr(b_cmd, "numba", False) and last: + pair.update(first_call_s=last["first_call_s"], wall_s=last["wall_s"]) + if c_cmd is not None: + t_c, o_c = run_once(c_cmd) + if o_c != o_a: + print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") + return {"outputs_differ": True} + pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) + pairs.append(pair) + ratios = [p["ratio"] for p in pairs] + out: dict[str, object] = {"pairs": pairs, "median_ratio": statistics.median(ratios), + "min_ratio": min(ratios), "max_ratio": max(ratios)} + if c_cmd is not None: + out["median_ratio_vs_noslots"] = statistics.median(p["ratio_vs_noslots"] for p in pairs) + return out + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) @@ -304,7 +641,7 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) - ab = sub.add_parser("interleaved", help="CPython vs @compiled alternated A/B; reports median ratios") + ab = sub.add_parser("interleaved", help="CPython vs @compiled/Cython/numba alternated A/B; reports median ratios") ab.add_argument("--python", default=sys.executable) ab.add_argument("--pairs", type=int, default=11) ab.add_argument("--write", action="store_true") @@ -314,6 +651,9 @@ def main() -> int: help="directory holding the typedpython package; enables the @compiled column") p.add_argument("--typedpython-python", default=sys.executable, help="interpreter for compiling (needs pyrefly 1.3.2)") + p.add_argument("--tools-python", default=str(TOOLS_PY_DEFAULT), + help="interpreter of the uv venv holding Cython and numba (default: benchmarks/" + "typedpython/.venv/bin/python); enables the Cython and numba columns") m.add_argument("--repeat", type=int, default=5) m.add_argument("--force", action="store_true") m.add_argument("--write", action="store_true") diff --git a/build.gradle.kts b/build.gradle.kts index 63c01e66a..c1a07f833 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -34,17 +34,33 @@ gradle.projectsEvaluated { // (see `pluginManagement { includeBuild(...) }` in settings.gradle.kts), so its tasks never join // the root task graph -- `./gradlew tasks --all` from here lists none of them, and a coordinator // publishing only what the root build shows would silently skip it. This task does not change how -// any of the three components publish (each already has its own `maven-publish` config); it only -// gives the root build one entry point that reaches all three. +// any component publishes (each already has its own `maven-publish` config); it only gives the root +// build one entry point that reaches all of them (the Compose host included). tasks.register("publishAllToMavenLocal") { group = "publishing" - description = "Publishes python-multiplatform, python-multiplatform-ksp, and " + + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + "python-multiplatform-gradle-plugin (including its plugin-marker publication) to mavenLocal()." dependsOn(":python-multiplatform:publishToMavenLocal") dependsOn(":python-multiplatform-ksp:publishToMavenLocal") + dependsOn(":python-multiplatform-compose:publishToMavenLocal") dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) } +// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with every +// component (library, KSP processor, Compose host, Gradle plugin). Signed when MAVEN_GPG_PRIVATE_KEY +// is set (gradle/central-publishing.gradle.kts). +// It does not clean the directory (the plugin build is another build, so ordering cannot reach +// it): .github/scripts/release/central-bundle.sh empties it first. +tasks.register("publishAllToCentralStaging") { + group = "publishing" + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + + "python-multiplatform-gradle-plugin (with its plugin marker) to build/central-staging." + dependsOn(":python-multiplatform:publishAllToCentralStaging") + dependsOn(":python-multiplatform-ksp:publishAllToCentralStaging") + dependsOn(":python-multiplatform-compose:publishAllToCentralStaging") + dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishAllToCentralStaging")) +} + // The build's single Kotlin/Wasm npm lock (one workspace for every wasm module) lives next to the // version catalog instead of a root `kotlin-js-store/` (issue #110). KGP only allows this on the // root project; the `*Property` forms are the non-deprecated API from Kotlin 2.4 on. Mismatch diff --git a/docs/design/threading-and-abi.md b/docs/design/threading-and-abi.md index 22865f0cc..b29e7056a 100644 --- a/docs/design/threading-and-abi.md +++ b/docs/design/threading-and-abi.md @@ -5,17 +5,22 @@ Parallelism comes from **free-threaded CPython**, not from per-interpreter GIL sub-interpreters (AGENTS.md §12.8). -The library supports **two flavours**, chosen at build time: +Maintainer, #158 (2026-10-04): "Free-threaded is the default and the only build. The GIL build is +not kept as an option. Baseline is CPython 3.15t." Desktop moved first (leader, 2026-11-13: "flip the +desktop default to 3.15t now, before the #377 alpha03 release"); the other platforms follow, so today +the build differs by platform (SPEC T-1): -| Interpreter | Threading behaviour | Status | +| Platform | Interpreter | Threading behaviour | |---|---|---| -| `python3.X` (GIL) | Kotlin threads each attach a thread state and serialise on the GIL | default (`pythonFreeThreaded=false` in `gradle.properties`) | -| `python3.Xt` (free-threaded) | Each Kotlin thread talks to its own free-running Python thread | **3.14t works on desktop**, opt-in with `-PpythonFreeThreaded=true` | - -The interpreter version is pinned by `pythonVersion` in `gradle.properties` (3.14.7 at the time of -writing). Free-threading is **not** gated on 3.15t: the whole desktop suite runs on 3.14.7t -(236 tests, 0 failures, 1 skipped, ROADMAP §9; SPEC T-2). Only desktop has free-threaded -prebuilts; Android and iOS have none, so the opt-in is desktop-only. How the earlier "from 3.15t" +| Desktop JVM, `macosArm64`, `linuxX64` | `python3.15t` (free-threaded), the only desktop build | Each Kotlin thread talks to its own free-running Python thread | +| Android (ART, androidNative; arm64-v8a and x86_64), iOS | `python3.15t` (free-threaded), the project's own build, the only mobile build since #328 | Each Kotlin thread talks to its own free-running Python thread (not yet run on an emulator or simulator) | +| wasmJs | GIL until #175 | single-threaded | + +The per-platform fact is `embedsFreeThreadedPython` (commonMain `EmbeddedPythonBuild.kt`, an +`expect`/`actual`); `-PpythonFreeThreaded` is removed. The interpreter version is pinned by +`pythonVersion` in `gradle.properties` (3.15.0). Before #158 free-threading was a desktop opt-in, and +the whole desktop suite ran on 3.14.7t (236 tests, 0 failures, 1 skipped, ROADMAP §9; SPEC T-2); the +3.15t default has not yet been observed running the suite. How the earlier "from 3.15t" conclusion was reached, and why it did not hold, is in [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). @@ -93,13 +98,13 @@ Design for two artefacts, but keep the split shallow enough to collapse later. ## Prebuilt availability Desktop (macOS arm64/x86_64, Linux x86_64, Windows x86_64) has GIL and free-threaded builds on the -python-build-standalone releases; the build uses the `-freethreaded` archive when -`pythonFreeThreaded=true`. **Android (python.org) and iOS (Python-Apple-support) publish no -free-threaded build**, so free-threading stays a desktop-only opt-in, and enabling it by default -would split the threading model, and the object-lifetime and thread-state design layered on top, -across platforms. Switching the default would additionally need adequate free-threaded wheel -coverage for the C extensions users care about. Table as of the 20260807 release, kept in the -archive note above. +python-build-standalone releases; the build uses only the `-freethreaded` archive (#158). +**Android (python.org) and iOS (Python-Apple-support) publish no free-threaded build**, so the project +builds its own (#163, release `cpython-3.15.0t-mobile`), and since #328 the library embeds those on +Android, androidNative and iOS. Only wasm still has a GIL build (#175), so the threading model is split +between wasm and everything else. The maintainer accepted the wheel-coverage cost on #158 +("third-party cp315t / abi3t wheels lag the release for a while; pypackpack builds from sdist where +they are missing"). Table as of the 20260807 release, kept in the archive note above. ## Thread-state management diff --git a/docs/design/typedpython-parallel-loops.md b/docs/design/typedpython-parallel-loops.md index e86c3e06b..2834d69a6 100644 --- a/docs/design/typedpython-parallel-loops.md +++ b/docs/design/typedpython-parallel-loops.md @@ -30,8 +30,7 @@ checkable on the existing IR (`typedpython/ir.py`), and says where that rule giv (§7, decision 3). Thread creation cost is measured together with the threshold. 6. A cost rule decides, at run time, whether a region is worth its fixed cost; below the threshold the sequential code runs (§8). The threshold is measured, not guessed here. -7. v1 parallelises only functions the user marks (decision 5). The spelling of the marking is a design - detail settled at implementation. Widening to unmarked functions later is easy; narrowing is hard. +7. v1 parallelises only functions the user marks `@compiled(parallel=True)` (decision 5, #358). Widening to unmarked functions later is easy; narrowing is hard. ## 2. What exists today (the facts this design stands on) @@ -78,7 +77,7 @@ condition that failed (like `verify/array-open`), so users can see why. - The function holds a copy-in list array, a local array, or only scalars (a pure scalar loop, e.g. a reduction over `range`). A function whose loop touches a buffer array (`array.array`) is not parallelised (decision 1). -- The user marked the function (decision 5). +- The user marked the function `@compiled(parallel=True)` (decision 5, #358). - Not under `entry_globals` mutation: entry globals are read once at entry and are locals in the body, so they are read-only and fine. @@ -503,9 +502,11 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. measured together with the threshold (`TP_PAR_MIN_WORK`). 4. **Float `+` and `*` reductions are excluded in v1.** Not permanently: a fastmath opt-in (#153) awaits the owner's decision, and N-15 follows that decision. Until then they stay sequential. -5. **Only functions the user marks are parallelised in v1.** How the marking is spelled (decorator - argument, comment, other) is a design detail to settle at implementation. Widening later (implicit - parallelisation) is easy; narrowing later is hard, so v1 starts narrow. +5. **Only functions the user marks are parallelised in v1.** Marked with `@compiled(parallel=True)` + (maintainer decision 2026-10-30, #358: `@compiled` is the only compilation mark, and options are + keyword arguments on `@compiled`, matching `#332` `@compiled(fastmath=True)`; the earlier + `# typedpython: parallel` comment marker was removed). Widening later (implicit parallelisation) is + easy; narrowing later is hard, so v1 starts narrow. 6. **`min`/`max` reductions are in v1 only if exactly equal to Python's** (§6). If an implementation cannot match it, they move to a later step and v1 keeps the integer and bitwise reductions only. 7. **Thresholds are measured, not chosen here.** `TP_PAR_MIN_WORK` and the thread cap (default 8) are @@ -516,4 +517,4 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. - (Settled: free-threaded 3.15t is the only build, #158, so there is no GIL-build variant.) - Which devices the threshold is measured on (the Mac mini only, or also an Android device). The measurement itself is decided (item 7); the device set is not. -- The spelling of the marking (item 5), settled at implementation. +- (Settled 2026-10-30, #358: the marking is `@compiled(parallel=True)`.) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 1ae91c1c2..f1a8db590 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -62,7 +62,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음, 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부, 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` (내장 이름, import 없음) 또는 모듈 선언 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | +| 1 `compiled` | `@compiled` (내장 이름, import 없음) | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL (직접) / WGSL / PTX, 빌드 타임 (§5) | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -155,6 +155,16 @@ API 안정성을 확인한 뒤로 미룬다. 않고** 인터프리터에 남기며 이유를 진단으로 낸다. 생성 C 는 `tp_runtime.h` 헬퍼로만 메모리와 `PyObject` 를 다룬다. 증명 밖의 안전망: ASan/UBSan, 인터프리터와의 차등 테스트, IR 수준 퍼징. +차등 테스트의 위치 (#41 M2 기준, 경로는 `python-multiplatform-ksp/src/test/python/`): + +| 경우 | 테스트 | +|---|---| +| i64 오버플로 경계 | `test_runtime.py::test_checked_arithmetic_matches_bigint_and_deopts_exactly_on_overflow`, `test_neg_i64_deopts_only_for_int64_min`, `test_fuzz_checked_arithmetic_against_bigint`; 컴파일된 함수에서 `test_cgen.py::test_i64_overflow_in_a_pure_function_deopts_to_the_big_int_result`, `test_native_int_impure.py::test_binary_ops_at_the_i64_boundary_equal_cpython`, `test_min_floordiv_minus_one_promotes` | +| 음수 floor 나눗셈과 나머지 | `test_runtime.py::test_floordiv_i64_is_python_floor_division`, `test_mod_i64_has_the_sign_of_the_divisor`, `test_fuzz_floordiv_mod_i64`, `test_floor_mod_sign_and_zero_cases_spelled_out`, `test_fuzz_float_floordiv_mod` | +| 0 으로 나누기 | `test_runtime.py::test_int_division_by_zero_messages_are_cpythons`, `test_float_division_by_signed_zero_raises_cpythons_message`; `test_cgen.py::test_integer_division_by_zero_has_cpythons_message`, `test_native_int_impure.py::test_division_by_zero_after_a_side_effect` | +| NaN 과 inf | `test_runtime.py::test_float_division_family_is_bit_exact_with_cpython`, `test_int_float_comparison_named_hard_cases`, `test_math_named_error_cases`, `test_math_one_argument_fuzz_is_bit_exact` (NaN 은 "어떤 NaN", 그 밖은 IEEE 비트 비교); `test_pipeline.py::test_fastmath_keeps_nan_checks` | +| 벤치마크 출력 | `test_benchmark_outputs.py`: 다섯 벤치마크를 `run.py verify` 크기로 컴파일 실행해 인터프리터 출력과 비교, TypedPython CI 가 develop 의 매 push 마다 돌린다 | + ### 4.3.2 증분 컴파일 (SPEC N-10) 사용자 결정: "증분 컴파일도 지원하도록 해. 빌드 속도 너무 오래 안걸리게". 단위는 모듈. 키는 소스 해시 + @@ -162,6 +172,58 @@ API 안정성을 확인한 뒤로 미룬다. 인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. +pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시는 두 층으로 나뉜다(#113). +`pipeline.generate_project` 는 소스 → C 만 캐시한다: 모듈마다 `.c` 하나, 그리고 `tp_runtime.h` 가 있는 +디렉터리. 키는 소스 + 의존 모듈 인터페이스 해시 + 컴파일러 버전이고, C 는 플랫폼·컴파일러·플래그를 읽지 +않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 +캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. + +### 4.3.2a `emit-c` manifest (SPEC N-10, #25 1단계) + +Gradle 쪽(#25 2단계, python-multiplatform-gradle-plugin)은 TypedPython 을 이 CLI 로 돌리고 manifest 를 +pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행하지 않는다. + + python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE + +- `PATHS`: 파일 또는 디렉터리(재귀, `typedpython check` 와 같은 규칙). `--root`: 모듈 이름의 기준 + 디렉터리(ppp 패키지의 `python/` 루트에 해당). `--cache`: `generate_project` 의 소스 → C 캐시. + `--manifest`: 쓸 파일. 상위 디렉터리는 만든다. +- 종료 코드: 0 manifest 를 씀, 1 게이트 오류(manifest 를 쓰지 않음, 원인은 stderr), 2 게이트를 실행할 수 없음. + C 컴파일러는 실행하지 않는다. + +형식(버전 1). 키 이름은 pypackpack `docs/design/compile-slot.md` 의 Kotlin 필드 이름과 같다. + +```json +{ + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/.../typedpython/runtime", + "nativeModules": [ + {"moduleName": "app.physics.nbody", "file": "app/physics/nbody.py", + "sources": ["/abs/cache/.../nbody.c"], "includeDirs": ["/abs/.../typedpython/runtime"], + "defines": {}, "status": "rebuilt", + "skippedItems": [{"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "..."}]} + ], + "notNativeModules": [ + {"module": "app.util", "file": "app/util.py", "kind": "NOT_MARKED", "items": []}, + {"module": "app.text", "file": "app/text.py", "kind": "NOTHING_COMPILED", + "items": [{"name": "f", "file": "app/text.py", "line": 2, "message": "..."}]} + ] +} +``` + +- `nativeModules[i]` → `ExtensionModuleSource(moduleName, sources, includeDirs, defines)`. `file`, `status`, + `skippedItems` 는 보고용이다(일부 함수가 인터프리트로 남는 것은 경고가 아니다, compile-slot.md §4). +- `notNativeModules[i]` → `NotNativeModule(module, kind, items)`, `items[j]` → `SkippedItem(name, file, line, message)`. + `line` 은 없으면 `null`. +- 모든 경로는 절대 경로(`/` 구분). `file` 만 `projectRoot` 기준 상대 경로다. +- 패키지 `__init__.py` 는 어느 목록에도 넣지 않는다(compile-slot.md: TypedPython 은 `__init__` 을 넘기지 않는다). + 그 밖의 모듈은 정확히 한 목록에 들어간다. 순서는 입력 파일 순서(디렉터리는 정렬)다. +- `sources` 는 캐시 안의 `.c` 를 가리킨다. 바뀌지 않은 모듈은 같은 경로, 같은 바이트다. +- 버전을 올리는 변경: 키 제거나 의미 변경. 키 추가는 버전을 올리지 않는다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. @@ -205,6 +267,53 @@ Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 프레임이 끼면 그 프레임은 세지 않는다. 테스트는 두 경우 모두 "둘 다 예외를 내고 죽지 않는다" 만 단언하고 깊이를 기록한다(`test_recursion.py`). +### 4.3.3b 파이썬 깊이는 구간마다 한 번 잰다 (#125, 에이전트 제안, 확인되지 않음) + +4.3.3a 의 진입 검사는 컴파일 프레임마다 세 가지를 부른다: `PyThreadState_GetFrame` (`tp_python_depth`, +최상위 프레임 동일성 캐시를 위해), `Py_GetRecursionLimit`, `Py_EnterRecursiveCall`. 출처는 CPython +v3.15.0rc1 태그다. + +- **파이썬 깊이: 옮긴다.** wrapper 의 impl 호출 하나를 *구간* 으로 본다. 구간 안에서 컴파일 프레임 밑의 + 파이썬 프레임은 바뀌지 않는다. 사이에 도는 파이썬 코드(콜백, `__del__`, 시그널 처리기, redo)는 쌓은 + 프레임을 모두 걷고 C 로 돌아오고, 그 밑의 프레임은 위에서 C 가 도는 동안 끝날 수 없다. + `PyThreadState_GetFrame` (`Python/pystate.c:2120`) 이 보는 `_PyThreadState_GetFrame` + (`Include/internal/pycore_interpframe.h:280`, 미완성 프레임을 건너뜀)과 `PyFrame_GetBack` + (`Objects/frameobject.c:2398`) 의 결과도 그래서 같다. 구간의 첫 검사가 재고(`tp_seg_depth`), 이후 검사는 + 그 값을 쓴다. wrapper 는 impl 호출 바로 앞과 바로 뒤에서 값을 지운다(`tp_depth_seg_begin` / `_end`). + 앞의 리셋이 없으면 안쪽 wrapper 가 바깥 구간의 얕은 깊이를 쓰고, 뒤의 리셋이 없으면 바깥 구간이 안쪽의 + 깊은 깊이를 계속 쓴다. 뒤의 리셋 덕에 바깥 구간은 안쪽 wrapper 가 돌아온 뒤 다음 검사에서 다시 재는데, + 이것은 예전에 `tp_python_depth` 가 다른 최상위 프레임을 보고 캐시를 바꾸던 바로 그 지점이다. 그래서 + 캐시가 붙잡는 프레임 객체와 그 수명(아래)도 그대로다. 검사 값은 예전 값과 같은 수이므로 + RecursionError 가 나는 깊이와 프레임은 바뀌지 않는다. 스레드마다 `tp_tls` 에 있고, `count` 처럼 한 + 스레드의 C 스택에서 호출이 LIFO 로 포개진다고 가정한다(greenlet 처럼 C 스택을 바꾸는 경우는 `count` 도 + 이미 맞지 않는다). +- **재귀 한도: 옮기지 않는다.** `Py_GetRecursionLimit` (`Python/ceval.c:7`) 은 interp 의 값을 읽는다. + `Py_SetRecursionLimit` (`Python/ceval.c:14`) 는 stop-the-world 로 바꾸지만, 같은 스레드에서는 컴파일 코드의 + 어느 `Py_DECREF` 든 `__del__` 을, 어느 호출이든 콜백을 돌릴 수 있고 거기서 `sys.setrecursionlimit` 을 + 부를 수 있다. 인터프리터는 다음 프레임에서 새 한도를 본다(`_Py_EnterRecursivePy`, + `Python/ceval_macros.h:406`, `py_recursion_remaining` 은 `Py_SetRecursionLimit` 이 고친다). 바뀐 것을 알 공개 + 수단이 없으므로 프레임마다 읽는다. +- **C 스택 가드: 옮기지 않는다.** 3.15 의 `Py_EnterRecursiveCall` (`Python/ceval.c:3576` → + `_Py_EnterRecursiveCallTstate`, `Include/internal/pycore_ceval.h:233`) 은 계수기가 아니라 이 프레임의 스택 + 주소를 `c_stack_soft_limit` 와 비교한다(`_Py_MakeRecCheck`, 같은 파일 211행). 깊은 프레임일수록 주소가 + 다르므로 한 번만 부르면 한도를 올린 깊은 재귀가 C 스택을 넘는다. `Py_LeaveRecursiveCall` 은 빈 함수다 + (같은 파일 256행). + +**관측되지 않는 근거와 시험.** `python-multiplatform-ksp/src/test/python/test_depth_once.py`. 구현 전 빨강: +생성 C 구조 시험 둘(wrapper 의 리셋, 헤더의 `tp_seg_depth`; 2026-11-13 cgen 을 일반 Python 으로 돌려 관측), +`-DTP_TRACE_DEPTH` 의 `__tp_trace_frame_reads__` 가 `down(100)` 에서 1 (예전 101, 추정). 구현 전에도 +초록인 보존 시험: 한도 주변 모든 파이썬 깊이와 재귀 깊이에서 인터프리터와 비교, 콜백 안의 중첩 컴파일 +호출(안쪽과 바깥쪽), 재귀 도중 `__del__` 과 콜백이 한도를 낮추고 올리기, 4 스레드, `sys._getframe` 이 보는 +프레임, 프레임 수명. 변이 셋: 앞 리셋 없음, 뒤 리셋 없음, 한도를 첫 프레임에서만 읽기. 빌드와 실행은 CI +에서 한다(에이전트는 빌드하지 않는다). + +**기존의 차이 둘(이 변경이 만든 것이 아니고 바꾸지도 않는다, 확인할 점).** (1) 컴파일 프레임 위에서 +돌아가는 순수 파이썬 프레임은 `py_recursion_remaining` 만으로 검사되므로 컴파일 프레임을 세지 않는다. 그 +위치에서 한도에 닿으면 인터프리터보다 늦게 실패한다. (2) 캐시가 안쪽 구간의 최상위 프레임(콜백 g)을 강한 +참조로 쥐므로, g 가 돌아올 때 프레임 객체가 지역 변수를 넘겨받고(`take_ownership`, +`Python/frame.c:48`, `:119`) 그 지역은 바깥 구간이 다시 잴 때 죽는다. 인터프리터에서는 g 가 돌아올 때 +죽는다. 시험은 이 시점을 develop 그대로 고정한다. + ### 4.3.4 객체 비용: 가상 객체, refcount 생략, 영역 할당 (SPEC N-12, 설계) 사용자 지시 (2026-10-03): "Rust 쪽으로 더 가는 방법을 적용하되 cpython과 동일성을 보장할 방법을 찾아." @@ -215,6 +324,34 @@ GC 추적), refcount 증감, 해제(`subtype_dealloc`)를 그대로 치르기 더 가려면 객체를 아예 만들지 않거나(가상 객체), 증감을 빼거나(refcount 생략), 한꺼번에 할당·해제해야 한다(영역 할당). 셋 다 `id()`, `__del__` 시점, `gc` 관측과 부딪칠 수 있다. +**실제 객체(N-11)의 생성 비용 (#125, 2026-11-05).** 프로파일(`benchmarks/typedpython/results/2026-10-25-binary-trees-alloc-profile.md`) +가운데 관측을 바꾸지 않고 줄일 수 있는 것은 다음과 같이 처리했다. +- 호출마다 스레드 상태를 한 번만 읽는다(#273). +- `tp_new_fixed` 는 새 객체의 슬롯을 `PyMember_SetOne` 대신 release store 로 직접 채운다. `SetOne` 이 + `Py_T_OBJECT_EX` 슬롯에 하는 일(객체 critical section 안에서 `Py_XNewRef(v)` 의 release store, 그 뒤 옛 값 + `Py_XDECREF`)에서 critical section 과 옛 값 해제만 뺀 것이다. 옛 값은 `tp_alloc` 이 0 으로 채웠으므로 NULL 이다. + critical section 을 빼도 되는 근거: `tp_alloc` 부터 마지막 슬롯 저장까지 스레드 상태를 놓는 호출도, eval + breaker 검사도 없어서 safepoint 가 없다. 그래서 stop-the-world 가 필요한 `gc.get_objects()` / + `gc.get_referrers()` 는 채우기가 끝나기 전에 이 객체를 다른 스레드에 줄 수 없다. 오히려 기존 critical + section 은 경합 시 스레드 상태를 놓을 수 있어서, 반쯤 채운 객체가 보이는 실행이 가능했다(그것도 인터프리터의 + `__init__` 실행과 같은 상태라 유효했다). 슬롯의 종류·쓰기 가능·오프셋 범위는 `tp_class_capture` 가 init 에서 + 확인한다. 참조 수: 슬롯은 `Py_NewRef` 로 자기 참조를 갖는다(이것을 빼는 변이는 `test_cgen_classes.py` 에서 + segfault 로 잡힌다). 측정(참고용, load 24~27): binary-trees `check(make(18))` 의 CPU 시간 중앙값이 develop + 대비 1.08배 빠르다. 같은 부하에서 벽시계 시간은 한 회차 안에서도 2~22초로 흔들려 쓸 수 없었다. + +하지 않은 것과 이유: +- **타입별 freelist.** 해제된 객체의 메모리를 붙잡으려면 클래스의 `tp_dealloc` 를 바꿔야 한다. 이것은 + 2026-10-25 결정(#125 댓글)에서 미뤘다: 클래스가 뒤에 `__del__`, `__weakref__`, 서브클래스를 얻으면 바뀐 dealloc 이 + 관측된다. free-threaded 빌드에서는 해제된 GC 객체의 메모리를 다른 스레드가 lock-free 로 읽을 수 있어서 + (`_Py_TryIncref` 계열) CPython 은 mimalloc 의 지연 해제(QSBR)에 기댄다. 확장이 따로 붙잡는 freelist 는 이 보장 + 밖에 있으므로, 이 결정을 다시 열 때 그 근거도 함께 써야 한다. +- **GC 추적을 늦추기/생략.** 아직 사이클을 만들 수 없는 노드(자식이 `None`)를 추적하지 않는 방식은 + `gc.is_tracked`, `gc.get_objects()` 로 관측된다. 그리고 인터프리터가 나중에 슬롯에 자기 자신을 넣으면 + (`PyMember_SetOne` 은 추적을 다시 켜지 않는다) 사이클이 수거되지 않는다. 컬렉션 횟수를 줄이는 방식(할당 계수 + 묶기, 임계값 조정)은 컬렉션 시점을 바꾼다. 둘 다 SPEC N-12 의 "GC 시점" 판단에 속하므로 오너 결정 사항이다. +- binary-trees 의 `main` 은 정책(#150)으로 컴파일되지 않고 순수하지도 않아서(`print`, `sys.argv`), 실제 객체를 + 아예 없애는 N-12b 영역이 이 벤치마크에는 적용되지 않는다. + **동일성의 정의.** 컴파일된 코드의 모든 실행은 **어떤 유효한 CPython 실행과 구별할 수 없어야 한다.** 구별은 관측으로만 할 수 있으므로, 설계는 "관측 지점" 목록과 "관측 지점에 닿기 전에 실제 객체로 되돌린다"는 규칙으로 이루어진다. CPython 자체가 정하지 않는 것(스레드 전환 시점)은 CPython 이 고를 수 있는 실행 중 @@ -310,7 +447,8 @@ PyPy·GraalVM 과 같은 방식이다. N-11 클래스의 객체를 PyObject 대 eval breaker 처리를 그대로 돌린다. 배열 copy-in·전역 스냅샷을 쥔 함수가 실행 중이면 미룬다. 그 함수 안에서의 GIL 넘기기는 2단계(#154)이고, 방식은 결정 대기(#152). **비용 (2026-10-17, 조용한 창, develop 대비 교대 11쌍):** fannkuch +8.2% (중앙값 1.082, 범위 1.043~1.191), - nbody 0.995 (배열 스냅샷 함수는 폴링하지 않음). 루프 폴링을 뺀 실험판은 fannkuch 1.000 이므로 남은 비용은 + nbody 0.995 (배열 스냅샷 함수는 폴링하지 않음; 이 측정 당시. #335(2026-10-25) 이후 스냅샷 함수도 폴링하며 폴링마다 + 리스트를 되쓰고 다시 복사한다, SPEC N-8, #406). 루프 폴링을 뺀 실험판은 fannkuch 1.000 이므로 남은 비용은 전부 짧은 안쪽 루프의 매 반복 카운트다운이다. CPython 도 모든 back-edge 에서 eval breaker 를 검사하므로 같은 종류의 비용이고, 시그널·스레드 굶김과 3.15t 의 GC stop-the-world 차단을 고치는 대가로 받아들였다(리더 결정). range 루프의 묶음 폴링은 후속 이슈다. 원자료: `benchmarks/typedpython/results/2026-10-17-eval-breaker-*.json`. @@ -320,6 +458,116 @@ PyPy·GraalVM 과 같은 방식이다. N-11 클래스의 객체를 PyObject 대 이 셋은 N-12 이전부터 있는 경계도 포함한다. 1번이 결정되어 N-12 구현을 시작한다. 2번은 #141(1단계 완료), 3번은 #142. +#### 바. 관측을 바꾸지 않는 할당·해제·refcount 비용 (#125, 조사 2026-11-13) + +**제약 (리더 경유, 2026-11-13, 오너의 상시 의도):** "All three GC options listed before are REJECTED: +per-type freelist (replaces tp_dealloc, id reuse), deferring or skipping GC tracking (gc.is_tracked / +gc.get_objects observable), batching GC allocation counts or adjusting thresholds (changes when +collections run). The rule behind it: no GIL-only features (everything must work on free-threaded +3.15t), and no change to any observable Python behaviour (gc module, ids, weakrefs, __del__ timing, +collection timing, sys.getrefcount, tracemalloc, subclassing, later class changes)." + +> 이 제약의 "collection timing" 은 마.1 의 결정 (2026-10-04, 사용자: GC 수집 시점은 동일성 범위에서 뺀다)보다 +> 좁다. 이 절의 후보는 둘 다 만족하도록 골랐다(수집 시점을 바꾸는 후보는 모두 뺐다). 두 문장 중 어느 쪽이 +> N-12a/b 의 가상 객체에도 적용되는지는 이 조사가 정하지 않는다(리더에게 확인할 점). + +**출발점.** 프로파일(`benchmarks/typedpython/results/2026-10-25-binary-trees-alloc-profile.md`, 3.15.0rc1t, +macOS arm64)은 #273 과 #371 이전 것이다. 몫은 그때의 전체에 대한 비율이고, 아래의 기대 이득은 그 몫에서 어림한 +**추정**이다(측정 아님). 컴파일 함수는 `make`, `check` 그대로다(`test_class_methods.py` EXPECTED). +CPython 출처는 v3.15.0rc1 태그의 파일과 함수 이름이다. + +**후보별 결론.** + +1. **같은 할당 경로를 단계만 줄이기.** `PyType_GenericAlloc` (`Objects/typeobject.c`) 은 + `_PyType_AllocNoTrack` 에서 크기 계산(`_PyObject_VAR_SIZE(type, nitems+1)`, itemsize 0 이면 basicsize), + `_PyObject_MallocWithType`, `_PyObject_GC_Link` (= `record_allocation(tstate)`, 할당 계수), 헤더 뒤 전체 + `memset`, `_PyObject_Init` (3.15t 에서는 heap type 의 참조를 `_Py_INCREF_TYPE`, 즉 스레드별 표에 센다, + `Include/internal/pycore_object.h`) 을 하고 마지막에 `_PyObject_GC_TRACK` 한다. 고정 레이아웃 정확 타입에서 + 남는 중복은 `memset` 하나다(모든 슬롯을 곧 쓴다). 공개 API `PyObject_GC_New` + `PyObject_GC_Track` 은 + `gc_alloc` (`Python/gc_free_threading.c`) 로 같은 할당기·같은 크기·같은 `record_allocation`·같은 + `_PyObject_Init` 를 거치고 `memset` 만 없다. 그래서 gc, tracemalloc, 할당 계수 어느 것도 다르지 않다. + 다만 객체의 모든 바이트를 컴파일 코드가 써야 하므로 클래스의 모든 슬롯이 필드여야 한다. 기대 이득: + memset 17 샘플, 약 1% (추정). **하지 않음:** 이득이 작다. 직접 할당(`tp_alloc` 를 건너뛰고 헤더를 손으로 + 채우기)은 비공개 API 이고, `Py_INCREF(type)` 로 바꾸면 3.15t 의 스레드별 type refcount 와 달라져 + `sys.getrefcount(Node)` 의 회계가 바뀌므로 기각. + 할당 하나에 CPython 안에서 스레드 상태 읽기가 셋 이상 있다(`gc_alloc`/`_PyObject_MallocWithType` 의 힙, + `record_allocation`, `_Py_INCREF_TYPE`), 해제 하나에도 셋(`_Py_Dealloc` 의 trashcan 여유 확인, + `PyObject_GC_Del` 의 `record_deallocation`, `_Py_DECREF_TYPE`). tstate 를 받는 공개 변형이 없으므로 공개 + API 로는 줄일 수 없다. 프로파일의 `_tlv_get_addr` 몫은 macOS 의 TLS 접근 방식 때문에 큰 것으로 보인다(추정). +2. **스레드 지역 객체의 refcount 피하기(3.15t biased refcounting).** `Py_INCREF` (`Include/refcount.h`) 는 + 3.15t 에서 `ob_ref_local` 을 relaxed 로 읽고, immortal 이면 끝, 소유 스레드(`ob_tid == _Py_ThreadId()`, + arm64 에서는 레지스터 읽기)면 원자 연산 없이 저장, 아니면 `ob_ref_shared` 에 원자 덧셈이다. `Py_DECREF` 도 + 같고, 0 이 되면 `_Py_MergeZeroLocalRefcount` 가 `ob_ref_shared == 0` 이면 바로 `_Py_Dealloc` 한다. 이 스레드가 + 만든 노드는 이미 싼 경로를 탄다. 그래서 "스레드 지역이면 refcount 를 다르게" 는 연산 하나당 더 얻을 것이 + 없다. 얻을 것은 **증감 쌍 자체를 없애는 것**이고, 그 조건은 스레드 지역성이 아니라 "다른 참조가 그동안 + 살려 둔다" 이다(아래 구현). 스레드 지역 증명은 슬롯 값을 빌려 읽을 때만 필요하다(나., D5, 바뀌지 않음). +3. **임시 객체 없애기(N-12a 가상 객체).** binary-trees 가 이득을 못 보는 이유: `make` 의 객체는 반환으로 + 탈출하고, `check` 는 아무것도 할당하지 않는다. 두 함수가 만나는 곳은 `check(make(d))` 를 부르는 `main` + 뿐인데, `main` 은 비용 정책(#150)으로 컴파일되지 않고 순수하지도 않다(`print`, `sys.argv`). 그래서 N-12b + 영역의 host 가 없고(별도 벤치 `binary_trees_region.py` 가 그 경우를 잰다), `make` 와 `check` 를 합치거나 + (`check∘make` 를 트리 없이 세기) `check` 를 호출자에 inline 하는 것도 호출 지점이 인터프리터 안이라 할 + 수 없다. `main` 을 컴파일하면 벤치의 컴파일 함수 집합이 바뀌므로 이 조사의 범위 밖이다. +4. **해제를 타입의 tp_dealloc 그대로, 간접 호출만 줄이기.** 지금 경로: `Py_DECREF` → + `_Py_MergeZeroLocalRefcount` → `_Py_Dealloc` (`Objects/object.c`: tstate 읽기, 재귀 여유가 작으면 trashcan, + reftracer) → `subtype_dealloc` (`PyObject_GC_UnTrack`, 베이스 사슬, finalizer·weaklist 확인, `clear_slots` + 가 자식마다 `Py_DECREF`, 재추적) → `object_dealloc` → `tp_free = PyObject_GC_Del` → `_Py_DECREF_TYPE`. 간접 + 호출은 둘셋이고 비용은 그 안의 일이다. 같은 일을 컴파일 코드가 하려면 비공개 API(`_PyTrash_*`, + `_Py_DECREF_TYPE`)가 필요하고 `tp_dealloc` 를 우회하게 되어 2026-10-25 결정(미룸)의 문제와 같다. 해제 + 순서를 바꾸면(예: 재귀 대신 반복) 이후 할당의 주소 재사용, 곧 `id()` 값이 달라진다. **하지 않음.** +5. **스레드 상태 몫 가운데 #273 이 남긴 것.** 컴파일 프레임마다 `tp_enter_call` 이 `PyThreadState_GetFrame` + (`Python/pystate.c`: 프레임 객체를 얻고 `Py_XNewRef`, 호출 하나와 증감 한 쌍), `Py_GetRecursionLimit` + (`Python/ceval.c`: `_PyInterpreterState_GET()`, TLS 읽기), `Py_EnterRecursiveCall` (TLS 읽기와 스택 주소 + 비교, `pycore_ceval.h` `_Py_EnterRecursiveCallTstate`) 를, 나갈 때 `Py_LeaveRecursiveCall` (3.15 에서 빈 함수) + 을 부른다. + - **파이썬 깊이:** 컴파일 프레임이 도는 동안 그 밑의 파이썬 프레임은 바뀌지 않는다(사이에 도는 파이썬 + 코드는 쌓았던 프레임을 다 걷고 돌아온다). 그래서 wrapper 가 한 번 재고 `tp_tls` 에 "구간 깊이" 로 두고 + 나갈 때 되돌리면, 프레임마다의 `PyThreadState_GetFrame` 이 없어진다. 첫 측정이 프레임 객체를 이미 + 만들고 붙잡으므로 새로 생기는 프레임 객체는 없다. 기대 이득: 프로파일의 `PyThreadState_GetFrame` 83 샘플, + 약 5% (추정, #273 전 몫). **에이전트 제안, 확인되지 않음.** `tp_tls`, wrapper, 빠른 진입(#147)을 함께 + 바꾸므로 이 단계에서는 하지 않았다. 다음 단계에서 구현: 4.3.3b (#412). + - **재귀 한도:** `Py_SetRecursionLimit` 는 3.15t 에서 stop-the-world 로, GIL 빌드에서는 GIL 을 쥐고 바꾼다. + 그래서 다른 스레드는 이 스레드가 safepoint 에 닿기 전에는 바꿀 수 없다. 하지만 이 스레드 안에서는 어느 + `Py_DECREF` 든 `__del__` 을 부를 수 있고, 거기서 한도를 바꿀 수 있다. 바뀐 것을 알 공개 수단이 없으므로 + 프레임마다 읽는 것을 그대로 둔다. + - **C 스택 가드:** tstate 를 받는 공개 변형이 없다. 몇 프레임마다만 부르면 C 스택이 먼저 닿는 경우 + RecursionError 가 나는 프레임이 달라진다. 그대로 둔다. +6. **기각된 것(위 제약).** 타입별 freelist, GC 추적 늦추기·생략, 할당 계수 묶기·임계값 조정. 참고로 3.15t 는 + 이미 스스로 할당 계수를 스레드별로 묶는다(`record_allocation`, `LOCAL_ALLOC_COUNT_THRESHOLD` 512). 이것은 + CPython 자신의 동작이고 컴파일 코드는 그대로 따른다. + +**구현한 것: 참조 옮기기와 빌린 피연산자 (#125, 에이전트 제안, 확인되지 않음).** 생성 C 에서 두 종류의 +1/-1 +쌍을 뺐다. 객체·할당·GC 추적·슬롯 값은 하나도 바뀌지 않는다. + +- **New 의 참조 옮기기.** `Node(make(d-1), make(d-1))` 는 인자 임시값을 `tp_new_fixed` 가 `Py_NewRef` 로 + 슬롯에 복사하고 호출자가 바로 놓았다. 이제 `tp_new_fixed_move` 가 임시값의 참조를 슬롯으로 옮기고 + 호출자는 임시값을 비운다(`_moved_temps`). 실패하면 아무것도 가져가지 않는다: 이미 쓴 슬롯은 반쯤 만든 + 객체를 놓기 전에 NULL 로 되돌린다. 비순수 함수의 느린 경로(`tp_call`)는 그대로 복사·해제한다. +- **빌린 피연산자.** Is 의 두 쪽, FieldGet·IsExact 의 객체, C 간 Call 의 OBJ 인자가 일반 OBJ 지역이면 복사 + (`Py_NewRef` + 해제) 없이 지역 이름(`l_x`)을, None 상수면 `Py_None` 을 바로 쓴다(`_borrow_operand`, + `_operand_release`). 지역의 자기 참조가 그 helper 가 끝날 때까지 객체를 살려 둔다. 이것은 이 프레임의 + 참조이므로 다른 스레드가 무엇을 해도 그렇다. IR 에는 대입식이 없어서 지역은 문장 경계에서만 바뀌고, + 그래서 helper 가 돌 때 읽는 지역은 피연산자를 평가할 때의 지역과 같다. 슬롯 **값**은 여전히 + `tp_field_get` (`PyMember_GetOne`) 으로 새 참조를 잡는다(D5 그대로). +- **관측되지 않는 근거.** 뺀 쌍은 모두 "다른 참조가 그동안 살려 두는 객체의 +1 뒤 -1" 이고, 그 사이에는 이 + 스레드에서 코드가 돌지도 safepoint 에 닿지도 않는다(New 의 근거는 위 #371 과 같다). 같은 스레드는 그 사이를 + 볼 수 없다. 다른 스레드(3.15t)가 그 사이에 `sys.getrefcount` 를 읽을 수 있었다면, 이제 읽는 값은 원래 + 실행에서도 그 창 밖에서 읽었을 값이다. 그래서 새 실행의 관측은 원래 실행에서 가능한 관측의 부분집합이다. + 호출 경계의 refcount, `gc.is_tracked`, `gc.get_objects`, `tp_traverse` 가 보는 슬롯, weakref, `__del__` + 순서, `id()` 는 변하지 않는다. 인터프리터 쪽 근거: 3.15.0rc1t 의 `dis` 로 `check` 를 보면 이 피연산자들이 + 모두 `LOAD_FAST_BORROW` 다(2026-11-13 관측). 인터프리터도 여기서 증감하지 않는다. +- **양.** binary-trees 의 `check` 한 번에 8쌍(FieldGet 의 `node` 2, Is 의 지역 2와 None 2, 재귀 인자 2), + `make` 의 노드 하나에 2쌍(자식 두 개, 잎에서는 None 두 개)이 빠진다. 기대 이득: 프로파일의 "compiled code" + 26% 가 이 증감을 포함한다. 3~8% 로 어림한다(추정, 측정 안 함). 측정은 CI 또는 조용한 창에서 리더가 한다. +- **남은 다음 단계(에이전트 제안, 확인되지 않음):** 대입되지 않는 OBJ 매개변수의 입양(`l_x = Py_NewRef(p_x)`, + 규칙 1)도 뺄 수 있다. 호출자가 호출 내내 인자를 살려 둔다(vectorcall 규약, C 간 호출은 호출자의 지역이나 + 임시값). `check` 한 번에 1쌍. 규칙 1 과 exit 블록을 바꾸므로 이 단계에서는 하지 않았다. +- **시험:** `python-multiplatform-ksp/src/test/python/test_alloc_preserving.py`. 생성 C 구조 시험 셋(구현 전 + 빨강), 인터프리터와 비교하는 관측 시험(`gc.is_tracked`, `gc.get_objects`, 살아 있는 동안 `id` 유일성, + weakref 콜백 순서, `__del__` 순서와 횟수, 뒤에 `__del__` 을 얻는 클래스, `__del__` 있는 서브클래스, 호출 경계의 + `sys.getrefcount`, 많은 호출 뒤 refcount, 4 스레드 동시 실행 뒤 refcount; 구현 전에도 초록, 보존을 지키는 + 시험), 변이 시험 셋(옮긴 임시값을 또 놓기, 옮기지 않고 복사하기 `-DTP_MUTANT_COPY_MOVED`, 빌린 + 지역을 놓기: 각각 refcount 나 `__del__` 시점으로 잡혀야 한다). + ### 4.3.5 소유권 추론, 메모리, 인터프리터 최소 상주 (설계, 결정 2026-10-05) 사용자 질문 (2026-10-05): "GC의 동작을 smart pointer로 전환하는 컴파일러를 개발하는게 가능할까?", 그리고 @@ -387,6 +635,126 @@ i64/f64 지역)는 **컴파일된 구간 안의 임시 값**만 줄인다. 컨 - 효과가 작아 하지 않는 것: `-OO`(0.35~0.8 MB, `assert` 도 사라짐), deopt 원본 지연 로딩(함수 1000 개당 약 0.4 MB), 동결 모듈 끄기(오히려 4.4 MB 증가). +### 4.3.6 리스트 매개변수 세 개 이상 (#255, 조사 기록과 제안) + +**출처.** +- 이슈 #255 본문: "User decision (2026-10-17): research option (C) for the nbody regression from #209." + 같은 본문: "Until (C) is settled, (B) stays." +- **리더 결정 (2026-11-13, 리더, 코디네이터 경유):** #255 는 리스트의 `ob_mutex` 에 기대지 않는다(critical + section API 전용으로 예약된 필드). 공개 critical section API 만 쓴다. +- 아래의 후보 판정과 (d1) 의 채택은 **에이전트 제안, 확인되지 않음**이다. + +**문제.** #209 이후 리스트 매개변수가 셋 이상인 함수는 3.15t 에서 해석 실행된다. copy-in 부터 write-back 까지 +모든 리스트의 잠금을 쥐어야 하는데, 공개 critical section API 는 객체 하나(`PyCriticalSection`) 또는 둘 +(`PyCriticalSection2`)만 잠근다. nbody 의 `offset_momentum`(리스트 4개), `advance`·`energy`(7개)가 31배에서 +1배로 떨어졌다. 기준은 동일성이다: 컴파일된 실행 하나하나가 CPython 의 어떤 interleaving 과 같아야 한다. + +#### 가. CPython 3.15.0rc1 의 근거 + +venv 의 3.15.0rc1t 헤더와 python/cpython 의 태그 `v3.15.0rc1` 소스(설치된 `pycore_lock.h` 와 바이트 단위로 같음을 +확인)에서 읽었다. 줄 번호는 그 태그 기준이다. + +1. **Begin.** 빠른 경로는 mutex 를 CAS 로 한 번 잡고 스레드의 섹션 스택에 올린다 + (`Include/internal/pycore_critical_section.h:106-117`). 실패하면 `_PyCriticalSection_BeginSlow` + (`Python/critical_section.c:20-62`): 같은 mutex 를 **맨 위** 섹션이 이미 쥐었으면 다시 잠그지 않고(29-45), + world 가 멈춰 있으면 잠그지 않고(46-54), 아니면 새 섹션을 먼저 올리고 `PyMutex_Lock` 으로 기다린다(55-60). 두 + 객체는 주소 순으로 잡고(`pycore_critical_section.h:163-170`), 느린 경로는 첫 mutex 를 쥔 채 둘째를 기다린다 + (`critical_section.c:91-101`). +2. **Detach 와 중단.** `PyMutex_Lock` 은 기다릴 때 스레드 상태를 놓는다(`Python/lock.c:654-656`; + `Python/parking_lot.c:328-337`). 놓을 때 활성 섹션 전부를 풀고(`Python/pystate.c:2285-2296`, + `critical_section.c:106-130`), 다시 붙을 때 **맨 위 하나만** 다시 잡는다(`pystate.c:2269-2273`, + `critical_section.c:132-165`). 그래서 세 번째 섹션을 중첩하면 경합 때 바깥 둘이 풀린 채 남는다 + (`Include/critical_section.h:16-22`: "This API does not provide a way to lock more than two objects at once"). +3. **섹션 구조체는 비공개.** "the contents of the structures are private" + (`Doc/c-api/synchronization.rst:225-232`, `Include/critical_section.h:39-40`). 스레드의 섹션 스택 + (`PyThreadState.critical_section`)을 읽는 공개 함수는 없다. `PyMutex_IsLocked` 는 잠겼는지만 알려 주고 누가 + 쥐었는지는 모른다. 3.14 의 `PyCriticalSection*_BeginMutex` 도 mutex 둘까지다 + (`Include/cpython/critical_section.h:38-42`). +4. **`ob_mutex`.** 문서: "reserved for use by the critical section API ... Do not lock it directly with + `PyMutex_Lock`" (`Doc/c-api/structures.rst:58-66`, `Doc/howto/free-threading-extensions.rst:391-410`). +5. **잠그는 CPython 연산과 잠그지 않는 읽기.** `PyList_SetItem` 은 리스트의 섹션을 연다 + (`Objects/listobject.c:454-479`). `list + list` 는 두 리스트를 함께 잠근다(`listobject.c:813`). `list.sort` 는 + key 를 부르는 동안 리스트의 섹션을 쥐고 리스트를 비워 둔다(`listobject.c:2925`, 2993). 원소 읽기 + `PyList_GetItemRef` 는 잠그지 않는 원자적 읽기이고(`listobject.c:354-379`, 405-419), 인터프리터의 `a[i]` 가 바로 + 그 읽기다(`Python/bytecodes.c:1148-1151`). dict 읽기는 실패 경로에서 dict 의 섹션을 연다 + (`Objects/dictobject.c:1647-1660`). `len` 은 원자적 길이 읽기다(3.15t `Include/cpython/listobject.h:30-36`). + +#### 나. 앞선 조사: `ob_mutex` 를 쓰는 방안 (리더 결정으로 제외) + +- **원안 (C), `PyMutex_Lock(&list->ob_mutex)` 를 주소 순으로: 건전하지 않다.** 호출자 섹션이 쥔 리스트 + (`lst.sort(key=lambda v: f(lst, b, c))`)에서 자기 교착한다: 기다리며 놓을 때 SuspendAll 이 그 섹션을 풀고, 다시 + 붙을 때 Resume 이 같은 mutex 를 잡으려 한다. 공정성 handoff 로 mutex 가 이미 우리 몫이면 그 Resume 은 영원히 + 기다린다(`critical_section.c:113`: 비활성 맨 위 섹션은 다시 풀지 않는다). 또 직접 쥔 채 기다리면 Resume 이 호출자 + 섹션의 mutex 를 주소 순서와 무관하게 잡아 순환이 생길 수 있고, write-back 의 `PyList_SetItem` 이 자기 스레드를 + 기다린다. +- **변형 (C'), 섹션 하나 + 나머지는 `ob_mutex` 시도만 (커밋 89e9bfae):** 위 결함은 피했지만 `ob_mutex` 를 문서의 + "reserved" 를 넘어 쓴다. 리더 결정(2026-11-13)으로 제외했고 코드에서 지웠다(`ob_mutex`, `PyMutex` 비트를 직접 쓰는 + 곳이 남아 있지 않다). + +#### 다. 공개 API 만으로 본 후보와 판정 + +- **(a) copy-in 전에 섹션을 중첩해 잡고, 하나도 중단되지 않았음을 확인: 불가.** 확인하려면 섹션 태그(`_cs_prev` + 의 INACTIVE 비트)나 스레드의 섹션 스택을 읽어야 하는데 둘 다 비공개다(위 3). 중단은 우리가 기다리는 순간(Begin + 의 느린 경로)에 일어나므로 시점을 피할 수도 없다(위 1, 2). +- **(b) 낙관적 검증: 리스트마다 자기 섹션 안에서 copy-in 하고 읽은 항목의 강한 참조를 쥔 뒤, 커밋 때 읽은 칸의 + 동일성을 다시 보고 write-back: 일반적으로 건전하지 않다.** + - 커밋이 원자적이지 않다. 섹션은 둘까지라, 세 리스트 이상에 쓰는 함수의 write-back 은 리스트별로 나뉜다. 예: + `a[0] = 1; b[0] = 1; a[0] = 2`. 인터프리터에서 다른 스레드가 볼 수 있는 상태는 (0,0), (1,0), (1,1), (2,1) 이다. + a 를 먼저 커밋하면 (2,0) 이, b 를 먼저 커밋하면 (0,1) 이 보인다. 둘 다 CPython 이 만들지 않는 상태이고, + `a + b`(두 리스트를 함께 잠근다, 위 5)나 2 리스트 컴파일 함수가 그것을 관측한다. 어떤 커밋 순서도 이것을 막지 + 못한다. 쓰는 리스트가 둘 이하면 `PyCriticalSection2` 하나로 원자적으로 커밋할 수 있다. + - 읽기 검증에는 한 순간이 필요하다. 쓰기와 프로그램 순서로 섞인 읽기는 커밋 순간의 값이어야 하는데, 커밋에서 + 잠글 수 없는 리스트는 그 순간에 볼 수 없다. 커밋 전과 후에 같은 객체를 다시 보는 것(double collect)은 ABA 를 + 막지 못한다: 다른 스레드가 `t = r[0]; r[0] = 5.0; r[0] = t` 로 같은 객체를 되돌릴 수 있고, 리스트에는 공개 + 버전 번호가 없다. + - 그러니 (b) 가 건전한 경우는 쓰기 구간 동안 손대는 리스트가 둘 이하인 경우뿐이고, 그것은 (c) 의 특수 경우다. +- **(c) 컴파일 타임에 리스트를 읽기 전용/쓰기 전용으로 나누거나 함수를 나눠 한 번에 섹션이 둘 이하가 되게: 쓰는 + 함수에는 지금 기반으로 불가.** + - 읽기 전용이라도 여러 리스트를 한 순간에 copy-in 할 필요는 남는다. 반례: 다른 스레드가 `a[1] = 1` 다음 + `b[0] = 1` 을 하고, 프로그램이 `a[0]`, `b[0]`, `a[1]` 순으로 읽는다. a 를 t1 에, b 를 t2 > t1 에 따로 copy-in 하면 + (`b[0]` = 1, `a[1]` = 0) 을 볼 수 있는데, 인터프리터는 `b[0]` 에서 1 을 보면 뒤의 `a[1]` 에서도 1 을 본다. + - 함수를 프로그램 지점에서 구간으로 나누고(각 구간이 리스트 둘 이하), 경계에서 write-back 하고 다시 copy-in 하는 + 것은 #152 의 poll 과 같은 논리로 건전하다. 하지만 경계에서 리스트가 바뀌었으면(길이, 원소 타입) 그 지점에서 + 인터프리터로 넘어가야 하고, 효과 뒤에는 deopt 할 수 없다. 지금 OSR 출구는 루프 poll 지점에만 있다. 임의 + 문장 지점의 연속(continuation)이 생기기 전에는 쓰는 함수에 쓸 수 없다(nbody `advance` 는 안쪽 루프 한 바퀴가 + 일곱 리스트를 모두 건드린다). +- **(d1) 순수 함수: copy-in 없이 원소마다 공개 API 로 읽기: 건전하다 (채택, 에이전트 제안, 확인되지 않음).** + - 순수 함수는 리스트를 읽기만 한다. 각 `a[i]` 를 `PyList_GetItemRef`(인터프리터의 읽기 그 자체, 원자적, 위 5) + 한 번으로, IR 의 프로그램 순서대로(문장마다 임시 변수에 순서대로 emit) 읽는다. 음수 색인은 그 읽기 때의 길이로 + 센다(인터프리터와 같음). `len(a)` 는 그 지점의 길이다. 이 실행은 인터프리터의 읽기를 같은 순서로 하는 것이므로 + 다른 스레드의 연산이 읽기 사이 어디에 끼든 그 interleaving 과 같다. 잠금을 쥐지 않으므로 교착도, 호출자 섹션 + 문제도, stop-the-world 지연도 없다. + - 원소 타입이 다르면(다른 스레드가 int 를 넣었다) deopt 한다. 순수 함수는 아직 아무 효과도 내지 않았으므로 + 해석 실행으로 다시 하는 것이 그대로 한 실행이다. 리스트가 줄었으면 그 지점에서 인터프리터와 같은 + `IndexError("list index out of range")`. 길이가 호출 중 바뀔 수 있으므로 증명된(검사 없는) 색인은 쓰지 않는다. + - 조건(verifier `verify/array-direct`, cgen 도 다시 거절): entry global 없음(입구에서 한 번 읽은 전역과 나중의 원소 + 읽기가 어떤 순서와도 안 맞을 수 있다, 위 (c) 의 반례와 같은 꼴), 버퍼 매개변수 없음, 리스트 매개변수의 + `a[:]` 없음(한 순간의 사본이 필요), 병렬 루프 없음(작업자는 Python API 를 쓰지 않는다). + - 리스트 하나, 둘의 경로는 그대로다: 같은 입력에 develop 과 바이트 단위로 같은 C 를 만든다(생성 C 비교로 확인). + - **비용(측정 전):** 읽기마다 함수 호출 하나와 참조 증감 한 쌍이다. copy-in 경로보다 느리고, 얼마나 빠른지는 CI + 측정을 기다린다. +- **(d2) 쓰는 함수의 원소별 접근: 불가(지금).** 쓰기는 `PyList_SetItem` 하나로 원자적이지만, 효과 뒤의 읽기가 다른 + 타입을 만나면 인터프리터는 계속 실행하는데 우리는 그 지점의 연속이 없다(위 (c) 와 같은 한계). 임의 지점 OSR 출구가 + 생기면 (d2) 가 nbody `advance` 를 푼다. + +#### 라. 판정과 결과 + +- 셋 이상의 리스트에 **쓰는** 함수는 (B) 그대로 해석 실행이다(develop 과 같은 frontend/verify/cgen 거절). +- **순수** 함수는 (d1) 로 컴파일된다. nbody: `energy` 가 컴파일되고, `advance`·`offset_momentum` 은 해석 + 실행이다(`test_class_methods.py` 의 EXPECTED). nbody 전체 속도는 `advance` 가 정하므로 거의 그대로일 것이다(측정 전). +- 시험(`test_free_threaded.py`): 순수 3·7 리스트 함수의 컴파일, 인터프리터와 같은 결과(빈 리스트, 별칭, 다른 타입 + 원소의 deopt, 리스트가 아닌 인자, 짧은 리스트의 IndexError); 쓰는 3 리스트 함수와 entry global 이 있는 3 리스트 + 함수는 해석 실행; 프로그램 순서 시험(쓰는 스레드가 `c[0]` 다음 `a[0]` 을 쓰고, `a[0]`, `c[0]` 순으로 읽는 함수가 + 늘 `c[0] >= a[0]` 을 본다); 동시에 int 와 float 를 바꿔 넣어도 결과가 늘 인터프리터의 것; `list.sort` key 에서 + 부르기. 변형 시험: 읽기 순서를 바꾼 변형(`race_mutant`, CI 기본 실행 밖)과 원소 타입 검사를 뺀 런타임(결정적, + CI 에서 실행)이 잡혀야 한다. 옵션 (A) 변형은 3 리스트 쓰기 경로가 없어져 대상이 없으므로 지웠다. + +#### 마. 오너 결정이 필요한 점 + +1. (d1) 을 받아들일지(에이전트 제안). 받아들이지 않으면 순수 함수도 (B) 로 돌린다(`ir.direct_lists` 하나). +2. 쓰는 함수(nbody `advance`)를 위해 임의 문장 지점의 OSR 출구를 만들지: (c) 의 구간 나누기와 (d2) 가 둘 다 그것에 + 걸려 있다. + ### 4.4 Kotlin 경계의 직접 호출 이 플랫폼에서만 가능한 이점이다. @@ -685,9 +1053,7 @@ upstream 대비 수치 오차와 커널 시간을 기준선으로 박아 둔다. 인터프리터 시작 시 `builtins.compiled` 를 설치하고(#42), 일반 CPython 에서는 `typedpython` wheel 의 `typedpython_builtins.pth` 가 `builtins` 에 없을 때만 항등 함수를 설치한다(런타임이 넣은 것을 덮어쓰지 않는다). 타입 검사기에는 `__builtins__.pyi` 로 알린다, Pyrefly 1.3.2 는 검색 경로의 그 파일을 추가 - 내장으로 읽는다(실측). 모듈 단위 표기는 `# typedpython: compiled` 주석을 유지한다: `# pyright: strict` 와 같은 - 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 - 모듈 전체를 지정할 수 있다. + 내장으로 읽는다(실측). 모듈 단위 pragma(`# typedpython: compiled`)는 2026-10-30 메인테이너 결정으로 제거했다(#353). `@compiled` 가 컴파일 대상을 표시하는 유일한 수단이다. 2. ~~고정폭 정수의 오버플로 의미~~, **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). 3. ~~pypackpack Cython 슬롯 인터페이스 소유~~, **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, C 를 낸 모듈 목록을 넘긴다. 확장이 원본 소스를 품으므로 별도 `.py` 폴백은 없다(Cython 폐기 후 갱신). 4. ~~GPU 착지를 torchnative 커스텀 op 로 할 것인가~~, **결정됨 (2026-10-04, 메인테이너):** 커널은 diff --git a/docs/guide/concepts.html b/docs/guide/concepts.html index de25f7f0c..48bf7fe17 100644 --- a/docs/guide/concepts.html +++ b/docs/guide/concepts.html @@ -74,7 +74,7 @@

03Lifetime across the boundary

CPython counts references; Kotlin collects garbage. A Kotlin wrapper takes one reference and gives it back when the wrapper is collected (a Cleaner, or a PhantomReference path on older Android). Cycles that run through a Kotlin proxy are made visible to Python's cyclic GC, so they are collected too.CPython 은 참조를 세고, Kotlin 은 가비지를 수거한다. Kotlin 래퍼는 참조 하나를 잡고, 래퍼가 수거되면 돌려준다 (Cleaner, 구형 Android 에서는 PhantomReference 경로). Kotlin 프록시를 지나는 순환도 Python 의 순환 GC 에 보이게 하므로 함께 수거된다.

04Threads스레드

-

Every C API call holds the GIL. Parallelism will come from free-threaded CPython (3.14t works on desktop with -PpythonFreeThreaded=true; the default is still the GIL), not from sub-interpreters: extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython(desktop 에서 -PpythonFreeThreaded=true 로 3.14t 동작, 기본값은 여전히 GIL)에서 온다. 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

+

Every C API call holds the GIL. Parallelism comes from free-threaded CPython, not from sub-interpreters (desktop, Android and iOS embed the free-threaded 3.15t and nothing else, with no switch; wasm still embeds the GIL build of 3.15 until its 3.15t runtime lands): extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython 에서 온다(desktop, Android, iOS 는 스위치 없이 free-threaded 3.15t 만 내장한다. wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다). 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

05Names이름

A Kotlin package is a Python module under the same name: androidx.compose.ui in Python is androidx. The binder never renames a namespace. Pythonic layers like pythonx are real Python packages that import those modules and reshape them, they live in their own repository.Kotlin 패키지는 같은 이름의 Python 모듈이다. Python 의 androidx.compose.ui 는 androidx 다. 바인더는 네임스페이스를 바꾸지 않는다. pythonx 같은 파이써닉 계층은 그 모듈을 가져와 다듬는 실제 Python 패키지이고, 자기 저장소에 있다.

diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html index 61f3d1b07..976e6e60b 100644 --- a/docs/guide/getting-started.html +++ b/docs/guide/getting-started.html @@ -34,7 +34,7 @@
Getting started시작하기

From clone to a running sample클론부터 샘플 실행까지

-

No release is published yet, so you build from source. The sample app is the shortest path to seeing both directions work.아직 배포된 릴리스가 없으므로 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

+

Releases are on Maven Central (section 04). To work on the library itself, build from source: the sample app is the shortest path to seeing both directions work.릴리스는 Maven Central 에 있다(04 절). 라이브러리 자체를 고치려면 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

01Requirements준비물

@@ -69,9 +69,62 @@

03Your first call

There is no close() in that code, and none is missing: every wrapper releases its Python reference when the Kotlin object is collected.위 코드에는 close() 가 없고, 빠진 것도 아니다. 모든 래퍼는 Kotlin 객체가 수거될 때 Python 참조를 놓는다.

-

04Verify a change변경 검증하기

-
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .tmp/build.log 2>&1; echo "EXIT=$?"
-./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .tmp/test.log 2>&1; echo "EXIT=$?"
+  

04Use it from Maven CentralMaven Central 에서 쓰기

+

Consumers must build with Kotlin 2.4.20-Beta2, the version the artifacts are compiled with. A desktop app that Python calls back into:소비자는 아티팩트가 컴파일된 Kotlin 2.4.20-Beta2 로 빌드해야 한다. Python 이 다시 호출하는 데스크톱 앱:

+
// settings.gradle.kts
+pluginManagement { repositories { mavenCentral(); gradlePluginPortal() } }
+dependencyResolutionManagement { repositories { mavenCentral(); google() } }
+
+// build.gradle.kts of a Kotlin Multiplatform app
+plugins {
+    kotlin("multiplatform") version "2.4.20-Beta2"
+    id("org.thisisthepy.python.multiplatform.bindings") version "3.15.0-alpha03"
+}
+kotlin {
+    jvm("desktop") {
+        @OptIn(org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi::class)
+        binaries { executable { mainClass.set("app.MainKt") } }
+    }
+    sourceSets.commonMain.dependencies {
+        implementation("org.thisisthepy:python-multiplatform:3.15.0-alpha03")
+        // Compose hosts (PythonAppView, PythonWidget, PythonLauncher):
+        // implementation("org.thisisthepy:python-multiplatform-compose:3.15.0-alpha03")
+    }
+}
+// A KMP executable is otherwise treated as a library and gets no FunctionTable.
+pythonBindings { role.set("app") }
+
// src/desktopMain/kotlin/app/Main.kt
+package app
+
+import python.multiplatform.ffi.Python3
+import python.multiplatform.ffi.upcall.PythonProxySource
+import python.native.ffi.UpcallStub
+
+class Greeter(val subject: String) {
+    fun greet(times: Long): String = List(times.toInt()) { "hello $subject" }.joinToString(", ")
+}
+
+fun main() {
+    Python3.initialize()
+    // Upcalls: install the generated table, bind the entry points, install the proxies.
+    python.multiplatform.generated.FunctionTable.installInto()
+    Python3.exec(
+        """
+        import ctypes
+        _pm_resolve = ctypes.CFUNCTYPE(ctypes.c_long, ctypes.c_char_p)(${UpcallStub.resolveHandleStubAddr})
+        _pm_invoke = ctypes.CFUNCTYPE(ctypes.py_object, ctypes.c_long, ctypes.py_object)(${UpcallStub.invokeWithArgsStubAddr})
+        _pm_release = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.releaseObjectStubAddr})
+        _pm_cancel = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.cancelCallStubAddr})
+        """.trimIndent(),
+    )
+    PythonProxySource.install()
+    Python3.exec("from app import Greeter\nprint(Greeter('Kotlin').greet(2), flush=True)")
+}
+

The library, the KSP processor, the Compose host and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix, and are published together on Maven Central; the current release is 3.15.0-alpha03. google() is needed for Android targets.라이브러리, KSP 프로세서, Compose 호스트, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓰고 Maven Central 에 함께 올라간다. 현재 릴리스는 3.15.0-alpha03 이다. Android 타깃에는 google() 이 필요하다.

+ +

05Verify a change변경 검증하기

+
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .cache/build.log 2>&1; echo "EXIT=$?"
+./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .cache/test.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform-ksp:fixtures:app:desktopTest --rerun
 ./gradlew :python-multiplatform-compose:fixtures:compose:desktopTest --rerun
 ./gradlew :python-multiplatform-gradle-plugin-fixtures:artifact:desktopTest --rerun
diff --git a/docs/guide/status.html b/docs/guide/status.html index d528808c4..414ec4393 100644 --- a/docs/guide/status.html +++ b/docs/guide/status.html @@ -50,7 +50,7 @@

What works today지금 되 Extension functions as methods확장 함수를 메서드로partial부분 .pyi stubs; klib binding.pyi 스텁, klib 바인딩partial부분 Linux and Windows desktop runsLinux, Windows 데스크톱 실행partial부분 -Free-threaded CPython 3.14t (desktop only, opt-in)free-threaded CPython 3.14t (desktop 한정, 옵트인)partial부분 +Free-threaded CPython 3.15t (desktop, Android and iOS: the only build; wasm still embeds the GIL build)free-threaded CPython 3.15t (desktop, Android, iOS: 유일한 빌드. wasm 은 아직 GIL 빌드)partial부분 Compose from Python on Android, iOS, wasmAndroid, iOS, wasm 에서 Python 으로 Composeplanned계획

@@ -60,7 +60,7 @@

FAQ

Is it published on Maven Central?Maven Central 에 배포되어 있나?

Not yet. Build from source until a release is announced.아직 아니다. 릴리스가 공지될 때까지 소스에서 빌드한다.

Which Python version?어떤 Python 버전인가?

-

CPython 3.14 today (pythonVersion in gradle.properties). Free-threaded builds can be selected with -PpythonFreeThreaded=true; this works on desktop only (236 tests, 0 failures); Android and iOS have no free-threaded prebuilts.현재 CPython 3.14 다 (gradle.properties 의 pythonVersion). free-threaded 빌드는 -PpythonFreeThreaded=true 로 고를 수 있고, desktop 에서만 동작하며(236 테스트 0 실패) Android/iOS 에는 free-threaded 프리빌트가 없다.

+

CPython 3.15 today (pythonVersion in gradle.properties). Desktop, Android and iOS embed the free-threaded build (3.15t) and have no GIL option; the old -PpythonFreeThreaded switch is removed and the build fails when it is passed. Wasm embeds the GIL build of 3.15 until its 3.15t runtime lands.현재 CPython 3.15 다 (gradle.properties 의 pythonVersion). desktop, Android, iOS 는 free-threaded 빌드(3.15t)만 내장하고 GIL 옵션이 없다. 예전의 -PpythonFreeThreaded 스위치는 없어졌고 넘기면 빌드가 실패한다. wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다.

Do C extensions work?C 확장이 동작하나?

Yes, in principle: this is CPython itself, not a re-implementation. Each extension still has to be built for each target platform.원리적으로 그렇다. 재구현이 아니라 CPython 그 자체이기 때문이다. 다만 확장마다 타깃 플랫폼별로 빌드되어 있어야 한다.

Why not reflection?왜 리플렉션을 쓰지 않나?

diff --git a/docs/investigations/fork-desktop-paths-macos.md b/docs/investigations/fork-desktop-paths-macos.md index bbdc507c1..df39c1995 100644 --- a/docs/investigations/fork-desktop-paths-macos.md +++ b/docs/investigations/fork-desktop-paths-macos.md @@ -2,128 +2,90 @@ What python-multiplatform needs before a pythonx-compose app runs on the two desktop paths `compose-multiplatform-core-extended` and `compose-multiplatform-extended` add: a GraalVM native -image and a Kotlin/Native executable, macOS arm64 first. Read on 2026-10-17 from the fork's -`origin/extended` branches (core `61906995435`, plugin fork `fca1473a`), `~/.m2/repository`, and -this repository's `develop` (`af4debac`). Nothing here was built; every fact cites where it was read. - -## 1. What the fork has on macOS +image and a Kotlin/Native executable, macOS arm64 first. Originally read on 2026-10-17 and refreshed +on 2026-11-01 after the fork published to Maven Central (#257, #287), the notebook E2E native-image +milestone landed (#246, #249, #251), and the `#191` decisions were recorded (2026-10-25). + +## 1. What the fork publishes on Maven Central + +The fork publishes its Compose libraries, window modules and Gradle plugins to Maven Central under +`org.thisisthepy.compose`: + +- **Libraries (`1.11.1-ext.2`)**: `org.thisisthepy.compose.runtime:*`, `ui:*`, `foundation:*`, + `material:*`, `material3:*`, `animation:*`, `components:*`, `org.thisisthepy.compose.androidx.lifecycle:*`, + including `macosarm64` and `linuxx64` klibs alongside `desktop` (JVM), `android`, `ios*` and + `wasmjs`. (`material-icons-core` stays on JetBrains `1.7.3`.) +- **Window modules (`1.11.1-ext.2`)**: `org.thisisthepy.compose.window:common`, `scene`, + `graalvm-macos`, `graalvm-linux`, `native-macos`, `native-linux`. +- **Gradle plugins (`1.11.1-ext.3`)**: `org.thisisthepy.compose` and + `org.thisisthepy.kotlin.native.desktop` (`compose-gradle-plugin:1.11.1-ext.3` on Maven Central). +- **Path selection in the fork's plugin**: `compose.desktop.application { output.set(ApplicationOutput.NativeImage) }` + (with `windowing.set(ApplicationWindowing.Awt)` or `AwtFree`, and `nativeImage { ... }` registering + `packageNativeImage`, `runNativeImageAgent`, `checkNativeImageNoAwt` and `packageApplication`), or + `compose.desktop.nativeApplication { ... }` for Kotlin/Native (`createDistributableNative*`, + `runNative*`, `packageKotlinNative`). Per decision 5 on #191, consumers select the path through the + fork plugin's setting directly; `python-multiplatform` adds no duplicate path-selection DSL. | | GraalVM native image | Kotlin/Native | |---|---|---| -| Fork's claim | `extended/README.md` (core, line 12): "macOS works on both: the Kotlin/Native window (AppKit and Metal) and the GraalVM native-image window." | same line | -| Window layer | `extended/window/graalvm/graalvm-macos` (JVM lib, AppKit through `@CFunction`, `upcalls.txt` table, no reflection) | `extended/window/native/macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`) | -| Compose host | **none for this path yet**: `extended/window/scene` (JVM and `macosArm64`) hosts a Compose scene through `runComposeWindow(platform, config, surfaceFor, content)` (`scene/src/.../ComposeWindow.kt:285`), but `graalvm-macos` provides no `WindowSurface` to draw into | `scene` with `native/macos`'s `MetalSurface` | -| Measured with Compose content | `extended/native-image/hello` (plugin fork): Material 3 in an **AWT** window (`windowing = Awt`), 82 MB, "draws the same content through Skia offscreen into a PNG byte for byte the one the JVM run draws" (`extended/native-image/README.md`, 2026-10-02) | no Gradle demo in the fork; compose-rust's Amper renderer (`darkpyonix/compose-rust/renderer/macos`) consumes `native-macos` | -| AWT-free demo | `extended/native-image/hello-awtfree-link`: opens, pumps and presents an AppKit window; **draws no Compose** | none | -| Build | the plugin fork's `org.jetbrains.compose` `1.11.1-extended-dev`: `compose.desktop.application { nativeImage { graalvmHome; skikoStaticDirectory } }`, `packageNativeImage`, `runNativeImageAgent`; skiko as a static archive (`extended/skiko/build-skiko-static-jvm.sh`) | `extended/gradle-plugin-kn`: `createDistributableNative...`, `runNative...`, `packageKotlinNative` | - -**Published to mavenLocal** (JetBrains coordinates kept, as #191 says): +| Window layer | `org.thisisthepy.compose.window:graalvm-macos`, `graalvm-linux` (JVM lib, `@CFunction` + `upcalls.txt` table, no reflection) | `org.thisisthepy.compose.window:native-macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`), `native-linux` (`linuxX64`) | +| Compose host | `org.thisisthepy.compose.window:scene` (`runComposeWindow`) | `scene` with `native-macos` / `native-linux` | +| Build | `org.thisisthepy.compose` `1.11.1-ext.3`: `compose.desktop.application { output.set(ApplicationOutput.NativeImage); nativeImage { ... } }`, `packageNativeImage`, `runNativeImageAgent` | `org.thisisthepy.kotlin.native.desktop` `1.11.1-ext.3`: `compose.desktop.nativeApplication { ... }`, `createDistributableNative*`, `runNative*` | -- `org.jetbrains.compose:ui*`, `foundation*`, `ui-text*` at `1.11.1001` (2026-10-17), including the - `macosarm64` klibs of `ui` and `foundation`. No `material3` macOS klib. -- `org.jetbrains.compose:compose-gradle-plugin:1.11.1-extended-dev`. -- `org.thisisthepy.compose.window:*:0.1.0`: `common`, `graalvm-macos`, `graalvm-linux`, `native-macos`. Not - `scene`, and not `native-linux`. -- `org.jetbrains.skiko:skiko-awt:0.144.6.1`. -- No publish script producing `1.11.1001` was found in either fork repository; the "Extended window - publish" workflow and compose-rust's `scripts/publish-window.sh` publish the window modules. - -## 2. What this repository has +## 2. What this repository has today +- **Switched to the fork's Maven Central artefacts (#257, #287).** `gradle/libs.versions.toml` uses + `org.thisisthepy.compose` `1.11.1-ext.3` (plugins) and `1.11.1-ext.2` (libraries). The artifact + walker (`ArtifactScanner.kt`) resolves `org.thisisthepy.compose.*` coordinates alongside + `org.jetbrains.compose.*`. - **Targets.** `python-multiplatform` declares `jvm("desktop")`, Android, iOS, androidNative and wasm; - no `macosArm64`, `linuxX64` or `mingwX64` (`python-multiplatform/build.gradle.kts`; the plugin's - `hostDesktopPlatform` knows macOS only as a JVM host). The sample has the same set. -- **A native image already runs Python** (`docs/platforms/graal-native-image-verification.md`, - 2026-08-17, `4687070a`): `:sample:nativeCompile` (an `Exec` of Liberica NIK 25.0.4's `native-image`, - `--no-fallback --enable-native-access=ALL-UNNAMED`) builds a headless `NativeImageMainKt`; 28 - downcalls and 5 upcalls registered, `UPCALL_OK`/`PROXY_OK`, no `libjvm`, `libpython` `dlopen`ed. - It is **headless on purpose**: the sample's `application {}` needs AWT. Not automated, not re-run. + `macosArm64` and `linuxX64` are the next interpreter targets to add (Phase A, decision 2; + `mingwX64` in Phase C). +- **A native image runs Python and stages `PYTHONHOME` beside the executable (#246, SPEC L-9).** + `pythonBindings.nativeImageDirectory` and `stageNativeImagePythonHome` copy the pruned + `python-build-standalone` prefix (`lib/python/**` and `lib/libpython.*`) into + `/python-multiplatform-home`. At run time, inside a GraalVM native image + (`org.graalvm.nativeimage.imagecode == "runtime"`), `PackagedPythonHome.resolve()` looks up + `/python-multiplatform-home` via `ProcessHandle.current().info().command()` + and hands it to CPython with `Py_SetPythonHome`. `:sample:nativeCompile` and `:sample:runNativeUpcallDemo` + stage and verify this layout with `JAVA_HOME` and `PYTHONHOME` unset (`docs/platforms/graal-native-image-verification.md`). - **The upcall table on a native image.** Desktop upcalls are FFM (`Panama.kt`), registered by the generated `META-INF/native-image/org.thisisthepy/python-multiplatform/reachability-metadata.json` - (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json`. That - metadata ships in the jar, so any native-image build that has the jar on its classpath picks it up. - (The verification note says no reflection registration is needed, while its own log shows 1,822 - types registered and the base file registers `UpcallTarget`; the note should be corrected.) -- **PYTHONHOME.** `stagePythonHome` sets `PYTHONHOME` for `JavaExec`/`Test` tasks and copies the prefix - into Compose's `prepare*AppResources`; the native demo is given `PYTHONHOME` by its run task. A - native image has no `java.home` and no packaged-resources directory, so nothing stages a prefix - **beside the executable** or finds it from there at start. -- **The notebook E2E** (`python-multiplatform-compose/fixtures/notebook-e2e`) needs no window: it - draws `PythonAppView` into an offscreen `ImageComposeScene` (`NotebookHost.kt:132`) and compares - pixels. It is a JVM `desktopTest` suite with `compose.desktop.currentOs` (Skia natives) and - `java.awt.headless=true`; no `java.awt`/`javax.swing` import. - -## 3. What each path needs from this side - -### GraalVM native image - -1. **A desktop app built by the plugin fork's `packageNativeImage`** rather than the sample's hand-made - `Exec`: the fork already solves Compose's own native-image configuration and static skiko. The - generated upcall table and FFM registrations come along in the jar's `META-INF/native-image`, so - **the upcall table carries over unchanged** (build-time table, no reflection: AGENTS.md §12.3, §13). - To check: that the fork's plugin passes `--enable-native-access=ALL-UNNAMED` (or lets an app add it) - and does not set `--no-fallback`-incompatible options that hide a missing FFM registration. -2. **PYTHONHOME beside the executable.** A task that stages the prefix and the installed wheels next to - `build/compose/native-image/main/`, and a runtime lookup (`ProcessHandle`'s executable path, a - sibling `python/` directory) when `PYTHONHOME` is unset, like `IosPythonHome` does for an app bundle. - **Staging does not carry over as is**: today's paths are a Gradle task's environment and the JVM - packaging's resources directory. -3. **The walker over the fork's jars.** Same public API, so the walk must produce the same bindings; - guarded by the stub diff of pythonx-compose #54. -4. **Selecting the path.** A Gradle plugin option (upstream Compose by default, the fork's native image - as a choice), #191's first bullet. - -### Kotlin/Native - -1. **An interpreter target**: `macosArm64` in `python-multiplatform` (cinterop against the macOS - python-build-standalone prefix the desktop JVM already downloads, a static or shipped - `libpython`), with the native `HandleTable`/trampoline path iOS and androidNative already use. - This is new work, not a switch. -2. **The klib walker** (B-2) over the fork's `macosarm64` klibs: since #234/#237 it carries handles and - interface members, but not constructors, properties or composables, and Compose's surface is mostly - composables and extensions; the K/N path needs the walker to reach about where the jar walker is. -3. **PYTHONHOME** beside the executable, as for the native image. -4. `material3` for `macosArm64`: no fork klib is published (the fork says only `ui` and `foundation` - differ from JetBrains; JetBrains' own `material3` macOS klib would have to resolve beside them). - -## 4. First milestone: the notebook E2E in a macOS GraalVM native image - -The E2E draws offscreen, so the milestone needs **no window** and none of the AWT-free pieces: - -1. A `notebook-e2e` native-image entry point (a `main` running the existing scenarios and printing a - pass/fail line per scenario, the way the fork's `hello` self-checks through `HELLO_SELF_CHECK`), - because a JUnit `desktopTest` does not run inside an image. -2. Built with the plugin fork's `packageNativeImage` (`windowing = Awt`, which is what the fork measured; - the scenes are offscreen either way), `--enable-native-access=ALL-UNNAMED`, the fork's static skiko. -3. PYTHONHOME and the pythonx-compose wheel staged beside the executable (§3, GraalVM 2). -4. Pass: the same 20 scenarios as the JVM run (pixel comparisons included), the binary run alone from - an empty directory with `JAVA_HOME` unset, and `otool -L` showing no `libjvm`. Then a CI job on a - macOS runner, as `notebook-e2e` has on the JVM (#213). - -Milestone 2 is the same app in the AWT-free window, once the fork has the pieces in §5. Kotlin/Native -follows the interpreter target (§3, K/N 1). - -## 5. Requests for the fork (through the darkpyonix lead) - -1. **Publish `org.thisisthepy.compose.window:scene`** (its `module.yaml` already enables publishing; - it is missing from mavenLocal and from the "Extended window publish" workflow's output). -2. **A `WindowSurface` for `graalvm-macos`** (Metal or GL into a Skia surface, as `native/macos`'s - `MetalSurface`), so `runComposeWindow` can host Compose content on the GraalVM path; and a demo that - draws Compose through it (`hello-awtfree-link` draws nothing today). -3. **A documented, reproducible publish of `1.11.1001`** (no script producing that version was found) - and one version line for the Compose libraries, the window modules and the Gradle plugin, so a - consumer pins one set. -4. **The native-image flags an app can add**: confirm that `packageNativeImage` lets an app pass - `--enable-native-access=ALL-UNNAMED` and merges a library's `META-INF/native-image` metadata (FFM - `foreign` registrations included). -5. **A hook to stage extra files beside the image** (a prefix directory), or confirmation that a - consumer task after `packageNativeImage` is the supported way. -6. For Kotlin/Native: whether `material3` for `macosArm64` is expected to come from JetBrains, and a - Gradle (not Amper-only) example of `scene` plus `native-macos` hosting Compose content. - -## 6. Open questions - -- Whether CPython's own native-image behaviour holds with the fork's Compose in the same image (one - process, Skia and libpython both `dlopen`ed or linked); not tried. -- Where the native-image E2E runs in CI: GitHub's macOS runners have no GraalVM by default, and the - fork's static skiko must be built or cached. + (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json` (which + registers `python.native.ffi.UpcallTarget`, corrected in #244). That metadata ships in the jar, so + any native-image build that has the jar on its classpath picks it up. +- **The notebook E2E runs in a macOS GraalVM native image (#249, #251, SPEC B-10).** + `python-multiplatform-compose/fixtures/notebook-e2e` builds `notebook-e2e` with `notebookE2eNativeCompile` + and runs all 20 `UI.ipynb` scenarios via `notebookE2eNativeRun` with `JAVA_HOME` and `PYTHONHOME` + removed, `compose.main.dispatcher=coroutines` (`NotebookE2eMainDispatcherFactory`, no AWT), and an + `otool -L` check forbidding `libjvm`. `.github/workflows/desktop.yml` (`notebook-e2e-native-image`) + runs this job on macOS against Maven Central. + +## 3. Remaining work by phase (per #191 decisions, 2026-10-25) + +### Phase A (in progress) + +1. **#251 (`desktop.yml`)**: `notebook-e2e-native-image` switched to Maven Central with the + `COMPOSE_EXTENDED_VERSION` guard and GitHub Packages check removed. +2. **`:sample` as a native-image output**: `:sample` sets `pythonBindings.nativeImageDirectory` so + `stageNativeImagePythonHome` stages `python-multiplatform-home/` beside `upcall-native-demo`, and + `runNativeUpcallDemo` runs the executable with `JAVA_HOME` and `PYTHONHOME` removed. +3. **Kotlin/Native desktop interpreter targets (`macosArm64`, `linuxX64`)**: add `macosArm64()` and + `linuxX64()` to `:python-multiplatform`, cinterop against the `python-build-standalone` headers + already downloaded by `downloadPython_macos_aarch64` and `downloadPython_linux_x86_64`, link and + load `libpython` dynamically beside the executable (decision 3), and wire `PackagedPythonHome` / + `PythonPayload` for desktop K/N executables. + +### Phase B + +1. **AWT-free GraalVM notebook app**: run the notebook app in an AWT-free window (`windowing = AwtFree`). +2. **Kotlin/Native macOS app**: re-check `KlibScanner` (B-2) coverage for `@Composable` functions, + constructors and properties over the fork's `macosarm64` klibs, add `macosArm64` to + `:python-multiplatform-compose`, and run a K/N macOS app. + +### Phase C + +1. **Windows (`mingwX64`)** once the fork publishes Windows artefacts. +2. **Stub diff guard (`pythonx-compose #54`)** and the final acceptance criterion: the same + `pythonx-compose` app running unchanged on upstream Compose, extended GraalVM and extended K/N + desktop builds with the notebook E2E passing in CI. diff --git a/docs/investigations/gc-scheduling-investigation.md b/docs/investigations/gc-scheduling-investigation.md index 5be003937..7da16ffcb 100644 --- a/docs/investigations/gc-scheduling-investigation.md +++ b/docs/investigations/gc-scheduling-investigation.md @@ -60,6 +60,13 @@ return and that the refcount did not move). ## 2. Free-threading: biased reference counting, and the queue only the eval loop drains +> **Update for 3.15.0 (2026-11-13, #398).** This section describes 3.14.7. In CPython 3.15.0, +> `_Py_brc_queue_object` gained a second path (`Python/brc.c:118`, `_PyThreadState_TrySuspendDetached`): +> when the owning thread is detached, the queuing thread merges the owner's queue on its behalf and frees +> what reaches zero. An embedder's thread is detached between `withGIL` scopes, so on 3.15t a cleaner's +> releases can be reclaimed without the owner reaching the eval loop. `EvalCheckpointTest` asserts the +> 3.15.0 behaviour. Whether this lowers the need for `autoDrainInterval` (ROADMAP 9.1) is not measured. + On a free-threaded build `Py_DECREF` branches on whether the thread owns the object (`ob_tid`): owner decrements `ob_ref_local` and may `tp_dealloc` at once; a non-owner decrements `ob_ref_shared` atomically, and if that takes it to zero the object is queued to its owner by @@ -188,6 +195,12 @@ the finding. ### 8d. Answered: the checkpoint is fine, the collector asks a second question +> **Update for 3.15.0 (2026-11-13, #398).** The table below is v3.14.7. In v3.15.0 the process-memory gate +> is gone (`gc_should_collect_mem_usage`, `get_process_mem_usage`, `last_mem` and `deferred_count` were +> removed): `gc_should_collect` (`Python/gc_free_threading.c:1996`) checks only the young threshold, +> `old[0].threshold == 0` (`:2004`) and `long_lived_total / 4` (`:2009`). `FreeThreadedGCGateTest` asserts +> that process footprint no longer moves the gate. The 8a to 8c measurements were not repeated on 3.15t. + Source-confirmed asymmetry (v3.14.7): | | GIL (`gc.c`) | free-threaded (`gc_free_threading.c`) | diff --git a/docs/investigations/typedpython-kernel-ir.md b/docs/investigations/typedpython-kernel-ir.md index 76d438456..5e566c90b 100644 --- a/docs/investigations/typedpython-kernel-ir.md +++ b/docs/investigations/typedpython-kernel-ir.md @@ -16,7 +16,7 @@ ### 1.1 MLIR Python 바인딩 wheel - [V] PyPI 의 `mlir-python-bindings` 는 **이름 선점용 빈 placeholder** (0.0.1, "not the real distribution"). `mlir` 도 2023 년 빈 패키지. 즉 `uv add mlir-python-bindings` 는 진짜를 주지 않는다 (`curl pypi.org/pypi//json`). -- [V] 실제 wheel 은 GitHub release `llvm/eudsl` 태그 `mlir-python-bindings` 에 있다 (GitHub API 로 목록 조회: 658 assets, 최신 `mlir_python_bindings-20261004+758aa4a92-*`, 거의 매일 갱신). 설치 가이드: `pip install mlir-python-bindings -f https://llvm.github.io/eudsl` (eudsl README 요약). 이전 저장소 `makslevental/mlir-wheels` 는 2025-11-29 archive, eudsl 로 이전됨 [V WebFetch]. +- [V] 실제 wheel 은 GitHub release `llvm/eudsl` 태그 `mlir-python-bindings` 에 있다 (GitHub API 로 목록 조회: 658 assets, 최신 `mlir_python_bindings-20261004+758aa4a92-*`, 거의 매일 갱신). 설치: `uv pip install mlir-python-bindings -f https://llvm.github.io/eudsl` (eudsl README 의 설치 안내를 uv 로 옮긴 것). 이전 저장소 `makslevental/mlir-wheels` 는 2025-11-29 archive, eudsl 로 이전됨 [V WebFetch]. - [V] 플랫폼/파이썬 매트릭스 (asset 이름 집계): cp310, 311, 312, 313, 314, **cp314t(free-threaded)** 각각 macOS arm64(`macosx_13_0_arm64`), Linux x86_64/aarch64(manylinux_2_34), Windows amd64. **cp315 / cp315t wheel 은 없음** (`cp315` 문자열 asset 0개). 크기 wheel 당 58 ~ 76 MB, 설치 후 `mlir/` 디렉터리 253 MB (macOS cp314t). - [V] 실제로 설치해 실행: `uv venv venv314t --python 3.14t; uv pip install --python venv314t/bin/python ` 성공. `sys.version` = "3.14.7 free-threading build", `import mlir` 후 `sys._is_gil_enabled()` = **False** (GIL 이 다시 켜지지 않음). 등록된 dialect: spirv, gpu, nvvm, vector, linalg, transform, amdgpu, x86, nvgpu, emitc, llvm, scf, memref, tensor, tosa 등 (`pkgutil.iter_modules(mlir.dialects.__path__)`). - [V] 공식 문서(mlir.llvm.org/docs/Bindings/Python/): free-threading 은 3.13+ 에서 "독립된 Context 라면 멀티스레드 안전". 안전하지 않다고 명시한 것: IR printing 과 PassManager 동시 사용, location error emit, module dump, transform interpreter, **GPU dialect 관련 작업**. 우리는 컴파일을 빌드 타임에 단일 프로세스/스레드로 돌리면 되므로 영향이 작다 [I]. diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md index fe74fad20..65a5e3d9d 100644 --- a/docs/locale/README_ko.md +++ b/docs/locale/README_ko.md @@ -8,7 +8,7 @@ [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](../../LICENSE) [![Kotlin Multiplatform](https://img.shields.io/badge/Kotlin-Multiplatform-7F52FF.svg?logo=kotlin&logoColor=white)](https://kotlinlang.org/docs/multiplatform.html) -[![CPython](https://img.shields.io/badge/CPython-3.14-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) +[![CPython](https://img.shields.io/badge/CPython-3.15-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) [![Platforms](https://img.shields.io/badge/platforms-JVM%20%7C%20Android%20%7C%20iOS%20%7C%20Android%20native%20%7C%20wasm%20(exp.)-lightgrey.svg)](#-플랫폼) [가이드](https://thisisthepy.github.io/python-multiplatform/) · @@ -196,7 +196,7 @@ Python → Kotlin 바인딩 코드가 없다. 자세한 내용: [설계 문서]( | 빌드된 jar 바인딩, Python 에서 Compose | ✅ 데스크톱 · ⏳ Android / iOS / wasm | | Kotlin/Native klib 바인딩 | 🟡 스캐너만 | | `.pyi` 스텁 생성 | 🟡 부분 | -| free-threaded CPython (3.14t, desktop 한정, `-PpythonFreeThreaded=true` 옵트인) | 🟡 부분 | +| free-threaded CPython 3.15t (데스크톱은 #158, Android / iOS 는 #328 이후 유일한 빌드; wasm 은 #175 까지 GIL) | 🟡 데스크톱 · 🟡 Android / iOS (컴파일만 확인, 에뮬레이터·시뮬레이터 실행 전) · ⏳ wasm | ## 🧱 생태계 diff --git a/docs/platforms/free-threaded-mobile.md b/docs/platforms/free-threaded-mobile.md index 8146231ae..4912b959d 100644 --- a/docs/platforms/free-threaded-mobile.md +++ b/docs/platforms/free-threaded-mobile.md @@ -1,19 +1,24 @@ # Free-threaded CPython for Android and iOS -Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI, -not yet run on a device or emulator.** Tracking issue: #163. +Status: built in CI and published as the GitHub release `cpython-3.15.0t-mobile` +(aarch64 Android run 39013371415, x86_64 Android run 43300549614 (#396), iOS run +39014667720); the SHA-256 of each archive is in the release's `SHA256SUMS` and pinned in +`python-checksums.properties`. Since #328 step 2 the library embeds these builds on Android +(arm64-v8a and x86_64), androidNative and iOS. **Not yet run on a device, emulator or +simulator.** Tracking issues: #163, #328. python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in -`AGENTS.md`). Desktop uses the python.org 3.14t builds. No prebuilt 3.15t exists for -Android or iOS, so the workflow `.github/workflows/build-ft-mobile.yml` builds it with +`AGENTS.md`). Desktop uses python-build-standalone's 3.15t builds. No prebuilt 3.15t exists +for Android or iOS, so the workflow `.github/workflows/build-ft-mobile.yml` builds it with CPython's own mobile tooling. ## What is built | Platform | Host | Tooling | Output | |---|---|---|---| -| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0-aarch64-linux-android.tar.gz` | -| iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0-iOS-XCframework.tar.gz` | +| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0t-aarch64-linux-android.tar.gz` | +| Android | `x86_64-linux-android` (#396) | `Platforms/Android` | `python-3.15.0t-x86_64-linux-android.tar.gz` | +| iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0t-iOS-XCframework.tar.gz` | The iOS tooling only builds an XCframework from all of its slices, so the simulator hosts are built as well as the device. There is no cheaper arm64-only path in the @@ -96,25 +101,41 @@ Each script fails the job unless: - the build Python reports `Py_GIL_DISABLED == 1`; - Android: `include/python3.15t/pyconfig.h` has `#define Py_GIL_DISABLED 1`, - `lib/libpython3.15t*.so` exists and is an aarch64 ELF, `lib/python3.15t` exists; + `lib/libpython3.15t*.so` exists and is an ELF of the host's architecture (aarch64 or x86-64), `lib/python3.15t` exists; - iOS: every slice has `Headers/pyconfig-.h` with `#define Py_GIL_DISABLED 1`, `lib/libpython3.15t.dylib`, and a `Python.framework/Python` binary (`lipo -info` is logged); the XCframework has `lib/python3.15t` and exactly two slices. The sha256 of each archive is written to the job summary and the archives are uploaded -as workflow artifacts (`python-t-android-aarch64`, +as workflow artifacts (`python-t-aarch64-linux-android`, `python-t-x86_64-linux-android`, `python-t-ios-xcframework`). +## How the library uses them (#328 step 2) + +- `python-multiplatform/build.gradle.kts` downloads the three archives from the release + (`downloadPython_android_aarch64`, `downloadPython_android_x86_64`, `downloadPython_ios`), + checks each against `python-checksums.properties` (`android-aarch64--freethreaded`, + `android-x86_64--freethreaded`, `ios--freethreaded`) and checks the unpacked headers: + `PY_VERSION` equals `pythonVersion`, and every `pyconfig` header defines + `Py_GIL_DISABLED 1` (the iOS `pyconfig.h` wrapper is skipped in favour of the + `pyconfig-.h` it includes). +- Android unpacks under `extracted//android--freethreaded/`, links androidNative + with `-lpython3.15t` against `include/python3.15t`, names the JNI library + `libmultiplatform_python3.15t.so` (what `manager.loadLibPython` asks for), and packages + `libpython3.15t.so` and `assets//lib/python3.15t/` into the AAR for arm64-v8a and x86_64. +- iOS compiles and links against the 3.15t `Python.xcframework`; the simulator test binary + gets `lib/python3.15t/` from `Python.xcframework/lib/python3.15t` and the slice's + `lib-arm64/python3.15t`. The plugin's iOS consumer wiring stages the same framework (SPEC T-2a). +- `embedsFreeThreadedPython` is `true` in `androidMain`, `artMain` and `iosMain`, so + `Versions.abiFlags` is `t` and the stdlib, stamp and library names follow. + ## Not verified -- The workflow has not run yet. Whether 3.15.0 configures and compiles with - `--disable-gil` for these targets is unknown; free-threaded support on Android and iOS - is not a documented CPython target. - Nothing has been loaded on a device, emulator or simulator. No import of `_ssl`, - `_ctypes` or extension modules, no GIL-off test run, no thread-safety check. + `_ctypes` or extension modules, no GIL-off test run, no thread-safety check. The tests + that would show it are `AndroidFreeThreadedBuildTest` (instrumented), + `AndroidNativeFreeThreadedBuildTest` and `IosFreeThreadedBuildTest`; they are compiled, + not run. - Whether the prebuilt dependency tarballs and the Xcode on `macos-15` suit 3.15.0 is - unconfirmed. -- Integration with the Gradle plugin's `PYTHONHOME` staging, with the Android - `libpython` loading code, and with the iOS framework embedding (which expects - `Python.framework` and a non-`t` library name in places) is not done. + unconfirmed beyond the build itself. - The x86_64 simulator slice is built only because the packaging step requires it. diff --git a/docs/platforms/free-threaded-wasm.md b/docs/platforms/free-threaded-wasm.md new file mode 100644 index 000000000..c3fe705dc --- /dev/null +++ b/docs/platforms/free-threaded-wasm.md @@ -0,0 +1,191 @@ +# Free-threaded CPython for wasm32-emscripten + +Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI.** Nothing +here has been compiled, linked or run. Tracking issue: #175 (this is its first step, the +build experiment). The owner approved the CI job on 2026-11-13 ("#175 may add the new wasm +3.15t CI build job"). + +python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in +`AGENTS.md`). The wasmJs target today runs a GIL, no-threads CPython 3.14.2 +([`wasm-design.md`](wasm-design.md)). No prebuilt free-threaded CPython with pthreads exists +for Emscripten, so the workflow `.github/workflows/build-ft-wasm.yml` builds one with +CPython's own driver and checks that it runs threads without the GIL under node. It does not +change `wasm.yml`, `python-multiplatform/scripts/wasm/build-cpython.sh` or the runtime the +wasmJs suite uses. + +## What is built + +| Host | Tooling | Output | +|---|---|---| +| `wasm32-emscripten`, pthreads, `--disable-gil` | `Platforms/emscripten` (CPython 3.15) | `python-3.15.0t-wasm32-emscripten.tar.gz` | + +The archive holds `python.mjs`, `python.wasm`, CPython's `node_entry.mjs` and `streams.mjs`, +`lib/python3.15t/` (the standard library without `test/`, plus `_sysconfigdata_*.py` and an +empty `lib-dynload/`), `include/python3.15t/` (with the host `pyconfig.h`) and +`BUILD-INFO.txt` (Emscripten version, flags, whether the trampoline patch was applied). The +driver has no install or package step for Emscripten, so the script assembles it. + +## Choices made for this experiment + +These are agent proposals from the #175 investigation, not decisions: + +- **Emscripten**: the version CPython itself pins (`emscripten-version` in + `Platforms/emscripten/config.toml`, 4.0.19 for 3.15.0), not the 5.0.3 that + `build-cpython.sh` pins for `pyemscripten_2026_0`. Which version the library should use is + open. The dispatch input `emscripten_version` overrides it, so 5.0.3 can be tried without + editing the script. +- **Node**: 24, the version `config.toml` pins, passed as `--host-runner node`. +- **Flags**: + + ```sh + python3 Platforms/emscripten configure-build-python -- --disable-gil + python3 Platforms/emscripten make-build-python + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-libffi + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-mpdec + python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread" \ + "LDFLAGS=-fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" \ + --disable-gil --enable-wasm-pthreads + python3 Platforms/emscripten make-host + ``` + + The driver puts `CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2` before user arguments and + autoconf keeps the last assignment, so the full string is repeated. `--enable-wasm-pthreads` + makes configure add `-pthread`, `-sUSE_PTHREADS` and `-sPROXY_TO_PTHREAD` itself. + `-sPTHREAD_POOL_SIZE=8` pre-creates Workers, because `Thread.start()` waits for the new + thread and a Worker cannot start while the JS thread that requested it is blocked. + `-sMAXIMUM_MEMORY=2GB` gives the shared memory a maximum, which shared memory requires. +- **libffi and mpdec** are rebuilt with `-pthread` through `EMCC_CFLAGS` (emcc appends it to + every invocation), because the driver builds them without it and wasm-ld is expected to + refuse non-atomic objects in a shared-memory link. +- The build Python is free-threaded too, as in [`free-threaded-mobile.md`](free-threaded-mobile.md). + +## The one source patch + +`Python/emscripten_trampoline.c` instantiates a wasm-gc call trampoline in pre-run. That +module is compiled with `-Wl,--import-memory` and no `-pthread` (`Makefile.pre.in`), so it +imports a non-shared memory, and instantiating it against the shared memory of a pthread +build is expected to throw outside the `try` block, at startup. Pyodide #237 reports this on +3.14, plus a second problem: the trampoline's `addFunction` pointer exists only in the main +thread's table. `build-wasm.sh` adds an early `return 0` to `getPyEMTrampolinePtr()` when +`wasmMemory.buffer` is a `SharedArrayBuffer`, which selects CPython's existing JS trampoline +(`wasmTable.get(func)` on the calling thread). The edit is asserted, so an upstream change +fails the build instead of skipping the patch. The dispatch input `trampoline_patch: skip` +builds without it, to observe the unpatched failure. + +No patch is applied for `--disable-gil` itself; the investigation found none needed. + +## Scripts + +`.github/scripts/ft-wasm/`: + +- `build-wasm.sh [VERSION]`: unpacks `Python-.tgz` (through + `.github/scripts/ft-mobile/common.sh`), installs Emscripten into `$FT_WORK/emsdk`, applies + the patch, runs the steps above, verifies, assembles and re-verifies the archive, and + writes its sha256 to the job summary. Honours `FT_WORK`, `FT_OUT` (default under `.tmp/`), + `FT_WASM_EMSDK_VERSION` and `FT_WASM_TRAMPOLINE_PATCH`. +- `smoke.py`: run under node by the script, see below. + +The build is meant for CI (`AGENTS.md` rule 10). The workflow runs on `workflow_dispatch` +and on pushes to `develop` that change its own files. + +## Verification built into the workflow + +The script stops at the first failed check, in this order: + +1. `emcc --version` reports the requested Emscripten version. +2. The build Python reports `Py_GIL_DISABLED == 1`. +3. `libffi.a` and `libmpdec.a` carry `+atomics` in their wasm `target_features`. +4. The host `pyconfig.h` has `#define Py_GIL_DISABLED 1`, the host Makefile has + `-sUSE_PTHREADS` in `CONFIGURE_LDFLAGS_NODIST` and `ABIFLAGS` is `t`. +5. `make-host` links `python.wasm` and runs `python.sh --version` (a startup failure, for + example the trampoline, fails here). +6. `python.wasm` has a shared memory with a declared maximum (read from its import or memory + section; the page counts are logged). +7. `smoke.py` under `python.sh` in the build tree: `sys.platform == "emscripten"`, + `Py_GIL_DISABLED == 1`, `sys._is_gil_enabled() is False`, then four `threading.Thread`s + each compute a sum; all must join within 120 s with the right results and four distinct + thread idents, and the GIL must still be off. It prints `FT-WASM-SMOKE-OK ...`, which the + script greps for. The run has a 600 s timeout (exit 124 means a thread start may have + deadlocked). +8. The interpreter's own `sysconfig.get_path("stdlib")` ends in `python3.15t`; the archive's + stdlib directory is named from it. +9. The unpacked archive has `include/python3.15t/pyconfig.h` with `Py_GIL_DISABLED 1`, + `lib/python3.15t/os.py` and `_sysconfigdata_*.py`, and `smoke.py` passes again with + `PYTHONHOME` pointing at it. + +The archive is uploaded as the workflow artifact `python-t-wasm32-emscripten`. No +release is published or changed. + +## Embedding probe (step 2 of #175) + +Agent proposal toward #175 step 2, not a decision. It answers, on CI, whether the library's way +of using CPython (load `python.mjs` as a factory with `noInitialRun`, never call `main`, call +exports from the JS main thread) works against this pthreads build. + +`build-wasm.sh` now appends `wasmExports,wasmMemory,wasmTable` to every +`-sEXPORTED_RUNTIME_METHODS` list in the host Makefile before `make-host` (the same edit as +`patch_makefile_reachability` in `python-multiplatform/scripts/wasm/build-cpython.sh`, as a +regex because the 3.15 Makefile text was not checked) and asserts the result. If the Makefile +has a `-sEXPORTED_FUNCTIONS=` list it also ensures `_malloc,_free,_Py_Version` are in it. A +missing list is only logged: whether 3.15 still exports C symbols through `-sMAIN_MODULE` +(as 3.14 does) is unverified, and the probe fails loudly if an export it needs is absent. + +The workflow step `Embedding probe (no main)` runs after the archive upload, so a probe +failure or hang does not lose the archive: +`timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs `. It prints one +`[x] PASS|FAIL` line per check and `FT-WASM-EMBED-PROBE-OK` only if all pass; on a timeout +(exit 124) the step reports the last check line printed. + +- `a`: the factory loads with `noInitialRun`, `Module.wasmMemory.buffer` is a + `SharedArrayBuffer` (page count printed). +- `b`: two hand-assembled modules import `intrinsics.memory` (the name Kotlin/Wasm uses). + A1 imports a non-shared memory and must fail with `LinkError`; A2 imports a shared one with + maximum 65536 pages and must instantiate, and `A2.load(&Py_Version)` must equal the value + read through a view of `wasmMemory`. A2 passing keeps the post-link import patch (investigation + options 2a/2b) open; A1 not failing would mean the blocker is not real. +- `c`: `Py_Initialize`, `PyEval_SaveThread`, then `PyGILState_Ensure` and `PyRun_SimpleString` + (strings placed with the exported `malloc`) of a script that starts four threads which print + and read `os.py`, joins them, asserts `sys._is_gil_enabled()` is False, and runs + `gc.collect()` inside a worker thread. This exercises the main thread blocking in `join` + while Workers need it to service proxied syscalls. + +Node only. Node's main thread uses a real `Atomics.wait`; a browser's main thread busy-waits +and services proxied calls, so a pass here does not show the browser works, and a Node deadlock +may not occur in a browser. It does not involve Kotlin at all (no upcalls, no proxies, no +Kotlin-side memory import). + +## Not verified + +- The workflow has not run yet. Whether 3.15.0 configures, compiles and links with + `--disable-gil --enable-wasm-pthreads` on Emscripten 4.0.19 is unknown. The only real + evidence is upstream's free-threaded build **without** pthreads (CPython gh-158997, on + `main` with Emscripten 6.0.10) and Pyodide's **GIL** build with pthreads (3.14, Emscripten + 5.0.3). No project is known to have combined the two. +- Whether `EMCC_CFLAGS=-pthread` is enough for libffi and mpdec, and whether the + `-sUSE_BZIP2` port links into a shared-memory build, is a prediction. +- Whether the JS trampoline fallback is enough, and what it costs per call compared with the + wasm-gc trampoline (`WasmCallTrampolineTest` covers only the current GIL build). +- Only node is exercised; no browser (which also needs cross-origin isolation for + `SharedArrayBuffer`). +- The smoke test checks that four threads run and join with the GIL off. It is not a + thread-safety test, does not run CPython's test suite, and does not measure parallel + speed-up. +- mimalloc's WASI path on Emscripten has no thread-exit hook; the investigation predicts a + slow per-thread leak. Not measured. + +## Open: the Kotlin side (later steps of #175, not implemented) + +- Kotlin/Wasm imports `intrinsics.memory` as non-shared (`min=0, max=none`); a shared + memory cannot satisfy that import, so the library cannot instantiate this `python.wasm` + until the import is marked shared with a maximum. That would revive the binary patch + [`wasm-design.md`](wasm-design.md) calls obsolete. +- Kotlin calls CPython on the JS thread that loaded the module (`noInitialRun`, no `main`), + so `-sPROXY_TO_PTHREAD` does not apply there; a first `Thread.start()` from Kotlin relies on + the pre-created worker pool. +- Upcalls from a non-main Python thread: the Kotlin trampolines are placed with `table.set` + only in the main thread's table; each Worker has its own table, so every Worker needs the + same entries (SPEC U-5, the pythonx-concurrent wasm dispatchers). +- Staging this archive as a wasmJs runtime (it lacks `wasmExports,wasmMemory` in + `-sEXPORTED_RUNTIME_METHODS` and the `pyemscripten` platform tag that `build-cpython.sh` + adds). diff --git a/docs/platforms/python-version-acquisition.md b/docs/platforms/python-version-acquisition.md index 544c13d3f..325fa57b5 100644 --- a/docs/platforms/python-version-acquisition.md +++ b/docs/platforms/python-version-acquisition.md @@ -23,41 +23,69 @@ The project currently uses `python-build-standalone` (maintained by Astral). For * **Naming:** The string `freethreaded` is injected into the asset name before the flavour, e.g., `cpython-3.14.7+20260807-aarch64-apple-darwin-freethreaded-install_only.tar.gz`. * **ABI and Libraries:** The ABI suffix is `t`, meaning the shared libraries will be named `libpython3.14t.dylib` or `libpython3.14t.so`. * **Availability:** They are available for all major desktop targets (macOS, Linux, Windows). +* **Acquired: the free-threaded build only, on desktop** (issue #158; maintainer 2026-10-04: "Free-threaded + is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t"; + leader 2026-11-13: "flip the desktop default to 3.15t now"). Every desktop `downloadPython_*` task fetches + `...-freethreaded-install_only_stripped.tar.gz` and unpacks it under `/-freethreaded/`. + The `-PpythonFreeThreaded` property that used to choose between the two is removed; passing it fails the + build. Android and iOS (section 3) embed the project's own 3.15t builds since #328. ## 3. Android and iOS Support -* **Android:** Prebuilt artifacts for Android are provided natively by python.org. These archives are plain NDK cross-compilations (e.g., CPython's own build tree). They can be found at `https://www.python.org/ftp/python//python---linux-android.tar.gz`. -* **iOS (3.14):** pinned to BeeWare `3.14-b11` (Python 3.14.7). `3.14-b10`, pinned earlier, shipped 3.14.6 while every other target shipped 3.14.7 (issue #47); b11 is the first 3.14 build with 3.14.7 (release 2026-09-04; its `VERSIONS` file and all four `patchlevel.h` say 3.14.7). The build now checks `PY_VERSION` in the extracted headers against `pythonVersion` for every target and fails if they differ. python.org's own iOS XCframework starts at 3.15 (section 5), so there is no alternative 3.14 source. -* **iOS:** Prebuilt artifacts for iOS are provided by the BeeWare project via their `Python-Apple-support` repository, which are CPython's own iOS build layouts (e.g. `prefix: iOS/Frameworks/...`). They are located at `https://github.com/beeware/Python-Apple-support/releases`. The kivy toolchains are **not** the source of these artifacts. -* **python-build-standalone Status:** `python-build-standalone` has moved from indygreg to Astral (`https://github.com/astral-sh/python-build-standalone`). It **does not** publish prebuilt binaries for `android` or `ios`. +* **Acquired: the project's own free-threaded 3.15t builds, and only those** (#328 step 2; maintainer on + #158, 2026-10-04: "Free-threaded is the default and the only build. The GIL build is not kept as an + option"; leader decision 2026-11-13 on #328: no opt-in flag on mobile; the owner approved the x86_64 + Android 3.15t build the same day, so arm64-v8a and x86_64 both stay supported). They are built by + `.github/workflows/build-ft-mobile.yml` (#163, `free-threaded-mobile.md`) and published on the GitHub + release `cpython-t-mobile` of `thisisthepy/python-multiplatform`: + `python-t-aarch64-linux-android.tar.gz`, `python-t-x86_64-linux-android.tar.gz` and + `python-t-iOS-XCframework.tar.gz`, at + `https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-t-mobile/`. +* **Layout.** Android: `prefix/lib/libpython3.15t.so`, `prefix/include/python3.15t/`, + `prefix/lib/python3.15t/` (with `config-3.15t--linux-android/`), unpacked under + `/android--freethreaded/`. iOS: `Python.xcframework//Python.framework/Headers` + (`pyconfig.h` wraps `pyconfig-.h`), `Python.xcframework/lib/python3.15t/` and + `/lib-/python3.15t/`, unpacked under `/ios/`. +* **Checks.** The lockfile pin (section 5), `PY_VERSION` against `pythonVersion` (issue #47), and + `Py_GIL_DISABLED 1` in every `pyconfig` header of the unpacked tree. +* **Not acquired any more:** python.org's GIL Android archives and GIL iOS XCframework (3.15), and BeeWare's + `Python-Apple-support` (3.14 and earlier, GIL). Neither publishes a free-threaded mobile build, and + the GIL build is not kept (#158). Their pins and the `pythonAppleSupportBuild` property are removed. +* **python-build-standalone Status:** `python-build-standalone` has moved from indygreg to Astral (`https://github.com/astral-sh/python-build-standalone`). It **does not** publish prebuilt binaries for `android` or `ios`. * **Prebuilt-cpython Status:** The `python/prebuilt-cpython` repository is currently a planning repository containing no artifacts, so it is not a source. -* **Conclusion:** We acquire Android artifacts directly from python.org, and iOS artifacts from BeeWare's Python-Apple-support, eliminating the need for kivy toolchain dependency. ## 4. Stable ABI and free-threaded builds * **The constraint:** the ~330 bindings target the CPython Stable ABI, and CPython's Limited API / `abi3` is not offered by free-threaded builds in 3.13 and 3.14 (3.15 adds `abi3t`, PEP 803). * **Why it does not block free-threading here:** the bindings are symbols resolved dynamically at run - time and `Py_LIMITED_API` is not defined, so no compile-time ABI contract applies. 3.14t works on - desktop with `-PpythonFreeThreaded=true` (the whole desktop suite: 236 tests, 0 failures, at the time - of ROADMAP §9; SPEC T-2). An earlier version of this section concluded the opposite ("fails or crashes + time and `Py_LIMITED_API` is not defined, so no compile-time ABI contract applies. 3.14t worked on + desktop when it was an opt-in (the whole desktop suite: 236 tests, 0 failures, at the time of ROADMAP + §9; SPEC T-2); desktop now embeds 3.15t only (section 2). An earlier version of this section concluded the opposite ("fails or crashes due to opaque structural changes", "wait for 3.15t") and was wrong; see [`../design/threading-and-abi.md`](../design/threading-and-abi.md) and [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). -* **Scope:** free-threaded prebuilts exist only for the desktop targets (section 2); Android (python.org) - and iOS (Python-Apple-support) publish none, so the flag is desktop-only and defaults to `false` - (`gradle.properties`). The interpreter version is `pythonVersion` in `gradle.properties`. +* **Scope:** free-threaded prebuilts from third parties exist only for the desktop targets (section 2); + Android and iOS use the project's own 3.15t builds (section 3, #163, #328), and wasm stays GIL until + #175. Which build each platform embeds is the library's per-platform `embedsFreeThreadedPython` + (SPEC T-1). The interpreter version is `pythonVersion` in + `gradle.properties`. ## 5. Security and Integrity Verification * **Desktop:** `python-build-standalone` provides a `SHA256SUMS` file with every release. The build script verifies the downloaded archive against this manifest. -* **Android (python.org) and iOS (python.org, 3.15+), Sigstore verified.** This bullet used to say that "full Sigstore verification directly inside Gradle is unreasonable without shelling out to external tools (like `sigstore-python` or the `cosign` CLI)". **That was wrong**, and it is now implemented. `dev.sigstore:sigstore-java` does the whole verification in-process, certificate chain, Rekor inclusion and OIDC identity, with no external binary. python.org publishes a sibling `.sigstore` bundle for both Android tarballs and, from 3.15, the iOS XCframework. (The extension is `.sigstore`; `.sigstore.json` returns 404.) There is still no plain checksum manifest, so the lockfile continues to pin the digest as well. -* **The signer identity is pinned, and has to be version-keyed.** Verifying a bundle *without* pinning an identity proves only that somebody holding a Sigstore certificate signed the bytes, which anyone can arrange. The build pins the Fulcio SAN and OIDC issuer of the actual CPython release manager, per . That must be a map rather than a constant, because it changes per release series: 3.14/3.15 are `hugo@python.org` via `https://github.com/login/oauth`, whereas 3.12/3.13 are `thomas@python.org` via `https://accounts.google.com`. An unrecorded series is a hard build failure, never a silent skip. -* **Opt-in, `-PverifyPythonSignatures=true`.** Off by default because `sigstore-java` pulls in grpc-netty-shaded, protobuf, bouncycastle and guava, and because fetching the TUF trust root needs network access, which would turn an offline build from working into failing. Gradle resolves the configuration lazily, so a default build downloads none of it (measured: a default `downloadPython_android_*` run mentions Sigstore zero times). +* **Android and iOS (the project's `cpython-t-mobile` release): the lockfile pin.** The release + carries a `SHA256SUMS` and no Sigstore bundle; the pins `android-aarch64--freethreaded`, + `android-x86_64--freethreaded` and `ios--freethreaded` were checked against it and against a + local `shasum` when they were written (2026-11-13). The build checks the lockfile only. +* **python.org's Android and iOS archives used to be Sigstore verified** through their sibling + `.sigstore` bundles, with the release manager's identity pinned per series. Those archives are + GIL builds and are no longer acquired (section 3), so that verification path and its identity map + were removed from the build. The record is in git history. +* **Opt-in, `-PverifyPythonSignatures=true`.** Off by default because `sigstore-java` pulls in grpc-netty-shaded, protobuf, bouncycastle and guava, and because fetching the TUF trust root needs network access, which would turn an offline build from working into failing. Gradle resolves the configuration lazily, so a default build downloads none of it. * **Desktop (`python-build-standalone`), Sigstore verified via GitHub Attestations.** The release carries 853 assets and the only non-archive among them is `SHA256SUMS`: there is no `.sigstore` file. Provenance is published through GitHub's *attestations* API instead, keyed by artifact **digest** rather than filename, and that endpoint is rate-limited to 60 requests/hour unauthenticated. Because of this rate limit and the bundle size (~212KB each), the build does not check these bundles into the repository. Instead, when `-PverifyPythonSignatures=true` is set, it fetches the DSSE bundle from the API, caches it locally as `.sigstore`, and uses `sigstore-java` to verify it against the pinned `astral-sh/python-build-standalone` workflow identity. Desktop is therefore protected by the lockfile, the release's own `SHA256SUMS`, and Sigstore. -* **iOS (BeeWare, ≤ 3.14; `3.14-b11` pin `b591f330...ab7b`, taken from GitHub's asset digest and recomputed locally), nothing exists to verify against.** The `Python-Apple-support` releases publish five `tar.gz` assets and nothing else: no checksums, no signatures, and no GitHub attestations. The lockfile pin is the only honest instrument available here, and no amount of build wiring changes that. * **The lockfile is not superseded by any of this.** The two gates prove different things, the lockfile says "these are the exact bytes this repository reviewed and pinned", Sigstore says "these are the bytes the release manager actually signed". The lockfile is also the only check that works offline and the only one covering every source, so it stays unconditional and Sigstore is layered on top of it. -* **The verification is known to fail when it should**, which is the only thing that makes it worth having. Two negative controls were run against the real 3.14.7 Android bundle: flipping one base64 character in `messageSignature.signature` gave `KeylessVerificationException: Artifact signature was not valid`, and pointing the identity map at the 3.13 release manager gave `No provided certificate identities matched values in certificate`. Both failed the build with a non-zero exit. +* **The verification is known to fail when it should**, which is the only thing that makes it worth having. Two negative controls were run against the real 3.14.7 Android bundle (python.org, before #328 removed that source): flipping one base64 character in `messageSignature.signature` gave `KeylessVerificationException: Artifact signature was not valid`, and pointing the identity map at the 3.13 release manager gave `No provided certificate identities matched values in certificate`. Both failed the build with a non-zero exit. ## 6. WebAssembly @@ -76,24 +104,28 @@ Compiling a C extension against the embedded CPython needs that CPython's header * **Key:** `(target, flavour)`. Targets: `macos-aarch64`, `macos-x86_64`, `linux-x86_64`, `windows-x86_64`, `android-aarch64`, `android-x86_64`, `ios-arm64`, `ios-arm64_x86_64-simulator` - (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.GIL` (default) or `FREE_THREADED` - (desktop only). + (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.FREE_THREADED` or `GIL`; the default + is the one the library embeds for that target, `CPythonIncludeLayout.embeddedFlavour(target)`: + `FREE_THREADED` on every target (desktop #158, Android and iOS #328). Android and iOS have no `GIL` + layout at all. (Making the default follow the embedded build, rather than `GIL` everywhere, is an + agent proposal, not confirmed: with `GIL` the documented `includeDir("macos-aarch64")` would throw + on every build.) * **Value:** `Provider` of the directory that directly contains `Python.h`, i.e. the `-I` directory: `/[-freethreaded]/python/include/python3.14[t]` on desktop (Windows has no - `python3.14` level), `/android-*/prefix/include/python3.14`, - `/ios/Python.xcframework//include/python3.14`. + `python3.14` level), `/android-*-freethreaded/prefix/include/python3.15t`, + `/ios/Python.xcframework//Python.framework/Headers`. * **Task dependency:** the provider carries the matching `downloadPython_*` task; using it as a task input makes Gradle download and extract first. -* **Only the configured flavour is extracted.** `-PpythonFreeThreaded=true` selects the free-threaded - desktop tree; asking for the other flavour throws a `GradleException` naming that property - (`isAvailable` tests it without throwing). +* **Only the embedded flavour is extracted.** Every target extracts the free-threaded tree only; asking for + the other flavour throws a `GradleException` naming #158 (desktop) or #328 (Android, iOS); + `isAvailable` tests it without throwing. ```kotlin // consumer build.gradle.kts evaluationDependsOn(":python-multiplatform") val includes = project(":python-multiplatform").extensions.getByType() tasks.register("compileExt") { - val inc = includes.includeDir("macos-aarch64") // or ("linux-x86_64", CPythonFlavour.FREE_THREADED) + val inc = includes.includeDir("macos-aarch64") // the free-threaded tree, .../include/python3.15t inputs.dir(inc) commandLine("cc", "-I${inc.get().asFile}", "-c", "ext.c") } @@ -107,27 +139,35 @@ The same extension exposes the library to link against (issue #56): | Target | Link library (under `/`) | |---|---| -| `android-aarch64`, `android-x86_64` | `/prefix/lib/libpython3.14.so` | +| `android-aarch64`, `android-x86_64` | `-freethreaded/prefix/lib/libpython3.15t.so` | | `windows-x86_64` | `windows-x86_64/python/libs/python314.lib` (`python3.lib` is the stable-ABI one) | | `windows-x86_64` free-threaded | `windows-x86_64-freethreaded/python/libs/python314t.lib` | | macOS, Linux, iOS | none -- `isLinkRequired(target)` is false; `libraryDir`/`libraryFile` throw `GradleException` | -Android has no free-threaded build, so that combination throws. `isLinkAvailable(target, flavour)` -tests without throwing; the other flavour fails naming `-PpythonFreeThreaded`, as for `includeDir`. +Android has no GIL build since #328, so that combination throws. On Windows the default is the +free-threaded `python315t.lib`; the GIL one is not extracted. `isLinkAvailable(target, flavour)` +tests without throwing; the other flavour fails naming #158, as for `includeDir`. ## Published version A consumer that builds against `python-multiplatform` must know which CPython it embeds (issue #61). -The answer is published three ways, all derived from `pythonVersion` / `pythonFreeThreaded` in -`gradle.properties`. +The answer is published three ways. The version is `pythonVersion` in `gradle.properties`; whether +the build is free-threaded differs by platform (desktop 3.15t since #158, Android, androidNative and +iOS 3.15t since #328, wasm GIL until #175), so each published artefact states its own. * **Extension** `pythonMultiplatform` (`python.multiplatform.gradle.EmbeddedPythonVersion`): - `pythonVersion` (`3.14.7`), `majorMinor` (`3.14`), `freeThreaded`. + `pythonVersion` (`3.15.0`), `majorMinor` (`3.15`), `freeThreaded`. One value per project: the + free-threaded one every non-wasm platform embeds (`true`; agent proposal, not confirmed). For any other platform, read its variant's + attribute or its resource. * **Gradle attributes** on every consumable `*Elements` configuration (so they land in the published module metadata): `org.thisisthepy.python.version` and `org.thisisthepy.python.free-threaded` - (both `String`; the latter is `"true"`/`"false"`). + (both `String`; the latter is `"true"`/`"false"`). The free-threaded one is per variant: `"true"` on + `desktop*`, `macosArm64*`, `linuxX64*`, `android*` (the AAR and androidNative) and `ios*` variants, + `"false"` on the wasm variants, and absent on shared metadata variants, which carry no CPython of their own (the + metadata rule is an agent proposal, not confirmed). * **Resource** `META-INF/python-multiplatform/python.properties` (`pythonVersion=`, `freeThreaded=`), - inside the desktop jar and the Android AAR's `classes.jar`. + inside the desktop jar (`freeThreaded=true`, from `generateDesktopPythonProperties`) and the Android + AAR's `classes.jar` (`freeThreaded=true` since #328, from `generateAndroidPythonProperties`). ```kotlin // composite build (includeBuild): read the extension @@ -142,6 +182,6 @@ val embedded = EmbeddedPythonVersion.parse(text) // published artefact: require a matching variant through the attributes configurations.named("desktopRuntimeClasspath") { - attributes.attribute(Attribute.of("org.thisisthepy.python.version", String::class.java), "3.14.7") + attributes.attribute(Attribute.of("org.thisisthepy.python.version", String::class.java), "3.15.0") } ``` diff --git a/docs/platforms/wasm-design.md b/docs/platforms/wasm-design.md index d0d385609..cd1eb3720 100644 --- a/docs/platforms/wasm-design.md +++ b/docs/platforms/wasm-design.md @@ -125,6 +125,10 @@ Details and the postcondition that guards the entry-module ordering are in the w - Whether compiled wheels other than pydantic-core load. - Compose (`@Composable` from Python) on wasm is `planned` (SPEC B-6, N-2); `suspend` upcalls are `planned` on wasm because there are no threads (SPEC U-5). +- Threads (#175): a free-threaded CPython 3.15t build with pthreads is a CI experiment + (`.github/workflows/build-ft-wasm.yml`), separate from the build above and not used by the library. + What it checks, and the Kotlin-side work it does not do (shared memory import, worker pool, + per-Worker function tables), is in [`free-threaded-wasm.md`](free-threaded-wasm.md). - Absolute wasmJs nanoseconds depend on suite scope and host lifetime (a filtered run is 55–74% slower in pure-Python rows); quote ratios, or full-suite figures labelled as such ([`downcall-design.md`](../design/downcall-design.md), "Warmup"). diff --git a/docs/release/desktop/typedpython/kernels.py b/docs/release/desktop/typedpython/kernels.py index 81cc9f952..3fb98914e 100644 --- a/docs/release/desktop/typedpython/kernels.py +++ b/docs/release/desktop/typedpython/kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import sys +@compiled def dot(n: int) -> float: s = 0.0 for i in range(n): @@ -10,6 +10,7 @@ def dot(n: int) -> float: return s +@compiled def collatz_steps(limit: int) -> int: total = 0 for start in range(1, limit): diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index bd328f959..85afd13a2 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -160,9 +160,20 @@ JNI callee names). Re-opening it means redesigning Android's convention selectio ## 9. Free-threading -3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (opt-in; `gradle.properties` -default `pythonFreeThreaded=false`): 236 tests, 0 failures, 1 skipped when last recorded, the same -count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is +Desktop (JVM, `macosArm64`, `linuxX64`) embeds the free-threaded CPython 3.15t and only that build +(maintainer, #158, 2026-10-04: "Free-threaded is the default and the only build. The GIL build is not +kept as an option. Baseline is CPython 3.15t"; leader, 2026-11-13: "flip the desktop default to 3.15t +now, before the #377 alpha03 release"). `-PpythonFreeThreaded` is removed; the build fails if it is +passed. Android (arm64-v8a, x86_64), androidNative and iOS embed the project's own 3.15t with no +opt-in flag (#328 step 2; leader 2026-11-13), compiled but not yet run on an emulator, device or +simulator (9.2); wasm is GIL (#175); the per-platform fact is `embedsFreeThreadedPython` (SPEC T-1). +The 3.15t default first ran the desktop suites in CI on 2026-11-13: Desktop CI +run 43306301208 counted `desktopTest` 647 tests, 3 failed, 1 skipped, and `linuxX64Test` 497 tests, 3 failed. +All six failures were tests asserting GIL-build or 3.14t behaviour (a `sys.getrefcount` that includes the +deferred-refcount bias, 3.15.0's detached-owner merge in `brc.c`, the removed GC memory gate, a probe +reading `ob_tid` as the refcount). They were corrected in #398 with no product change, and run 43329515729 +passed (Gradle prints no count on success). The earlier free-threaded record is 3.14t as an opt-in: 236 +tests, 0 failures, 1 skipped (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is drained by the eval-loop checkpoint (`PM/commonTest/.../ref/EvalCheckpointTest.kt`, `docs/investigations/gc-scheduling-investigation.md`); `FreeThreadedGCGateTest` guards the GC probe. The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done @@ -170,15 +181,23 @@ The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done free-threaded ones, are pinned in `python-checksums.properties`. - **9.1 `autoDrainInterval`'s default on the GIL build, open (decision).** Today - `if (BuildConfig.pythonFreeThreaded) 32 else 0` (`Python3.kt`). Off means a pure C-API embedder never + `if (embedsFreeThreadedPython) 32 else 0` (`Python3.kt`), so on by default on desktop, Android and + iOS (#328) and off only on wasm until #175; once wasm is free-threaded this question closes. Off means a pure C-API embedder never runs the cyclic collector: 20 000+ cyclic objects left with the automatic path off against ~1 970 with it on (`docs/investigations/gc-scheduling-investigation.md` §6–§7). On means a `__del__` or weakref callback can run at a `withGIL` exit the caller did not ask to yield from. The checkpoint costs ~128 ns over a 154.6 ns `withGIL` floor, ~4 ns amortised (`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`). A one-line change once decided. -- **9.2 Free-threading beyond desktop, open (blocked upstream).** No free-threaded CPython prebuilt +- **9.2 Free-threading beyond desktop, open (#328 run, wasm #175).** No free-threaded CPython prebuilt exists for Android (python.org ships `libpython3.14.so`/`3.15.so` only) or iOS (neither BeeWare - nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12. SPEC N-1. + nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12; the project builds its own (#163, + `.github/workflows/build-ft-mobile.yml`, release `cpython-3.15.0t-mobile`). #328 step 2 moved the + library: Android (arm64-v8a and x86_64), androidNative and iOS acquire those archives, link + `libpython3.15t`, package `lib/python3.15t`, report `abiFlags = "t"`, and publish + `freeThreaded=true`; the python.org and BeeWare GIL mobile archives and their pins are gone. Left: + an Android emulator run (`AndroidFreeThreadedBuildTest`, the instrumented suite, the androidNative + suite) and an iOS simulator run (`IosFreeThreadedBuildTest`, the ios-consumer fixture, sample DEMO + 1/8) with `Py_GIL_DISABLED` observed at run time; then wasm (#175). SPEC N-1. ## 10. WASM @@ -300,10 +319,17 @@ stripped), the root metadata jar is 382 KB, `PythonBootstrap` stages the Android desktop stdlib (`GP/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt`, SPEC L-4). Old §15a, §15b, §15d, §15f, §15g, §15h are closed; §15e item 4 is closed by §15h. -- **15.1 Nothing is published to a remote repository, open.** No publishing repository or signing is - configured (`signing { }` is commented out in `python-multiplatform/build.gradle.kts`), and - `repo1.maven.org/maven2/io/github/thisisthepy/python-multiplatform/` returns 404 (2026-10-03). - Every consumer verification so far used `mavenLocal()`. 10.1 depends on this or on a release asset. +- **15.1 Maven Central publishing, implemented, first upload pending (#375).** All three components + (the library for every target, the KSP processor, the Gradle plugin with its plugin marker) carry + Central POM metadata, sources and javadoc jars and in-memory GPG signing + (`gradle/central-publishing.gradle.kts`, signing only when `MAVEN_GPG_PRIVATE_KEY` is set, so + `publishToMavenLocal` still works without keys). `./gradlew publishAllToCentralStaging` fills + `build/central-staging`; `.github/scripts/release/central-bundle.sh` signs, checks and zips it, and + `.github/workflows/maven-central.yml` (version tag, or manual with a `dry_run` input) builds on macOS and + uploads one deployment to the Central Portal (`central-upload.sh`, environment `maven-central`). The + dry run signs with a throwaway key and uploads nothing. Open: the first real upload (needs the + secrets, only CI can show Central accepting the deployment), and `repo1.maven.org/maven2/org/thisisthepy/` + does not list the artifacts until then. 10.1 depends on this or on a release asset. - **15.2 The library, KSP processor and plugin publish under one version, done (#70).** Each build derives it from `gradle.properties` (`$pythonVersion-$libraryVersionSuffix`, a `-P` override included); the root build fails when a project's version differs, and `PublishedVersionTest` pins the processor diff --git a/gradle.properties b/gradle.properties index 000eaca4e..4dc862800 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,6 +21,8 @@ kotlin.native.binary.sourceInfoType=libbacktrace #Compose org.jetbrains.compose.experimental.jscanvas.enabled=true org.jetbrains.compose.experimental.uikit.enabled=true +# Compose gates its macosArm64 target behind this flag; :python-multiplatform-compose has it since #386. +org.jetbrains.compose.experimental.macos.enabled=true # org.thisisthepy.compose:compose-gradle-plugin:1.11.1-ext.3's Maven Central POM marks # hot-reload-gradle-plugin as optional (shadow configuration), so Gradle does not place it on the # plugin classpath unless disabled or declared separately. @@ -28,13 +30,11 @@ org.jetbrains.compose.hot.reload.disable=true # Python Build pythonVersion=3.15.0 -pythonFreeThreaded=false pythonBuildStandaloneRelease=20261013 -pythonAppleSupportBuild=b11 # Every published artifact (library, KSP processor, Gradle plugin) is versioned # "$pythonVersion-$libraryVersionSuffix"; each build derives it from these two values and fails if # its own version disagrees (#70). -libraryVersionSuffix=alpha02 +libraryVersionSuffix=alpha03 # GraalVM native-image (sample's nativeCompile task). Liberica NIK is not in a location Gradle's # toolchain auto-detection scans, so it is registered explicitly here. Adding a path only makes a @@ -46,7 +46,7 @@ org.gradle.java.installations.paths=/Volumes/macMini/toolchains/bellsoft-liberic # compose-multiplatform-core-extended (#191, #249): the Compose modules a native-image build takes from # the fork instead of upstream, and where from. Read in one place, by the notebook E2E's native image; # the fork's coordinates and version line are 1.11.1-ext.N, and only these lines follow. -composeExtended.version=1.11.1-ext.2 +composeExtended.version=1.11.1-ext.4 # The group the fork publishes under, as a prefix of upstream's (org.thisisthepy.compose). composeExtended.groupPrefix=org.thisisthepy.compose composeExtended.modules=org.thisisthepy.compose.ui:ui,org.thisisthepy.compose.ui:ui-desktop,org.thisisthepy.compose.ui:ui-text,org.thisisthepy.compose.ui:ui-text-desktop,org.thisisthepy.compose.foundation:foundation,org.thisisthepy.compose.foundation:foundation-desktop diff --git a/gradle/central-publishing.gradle.kts b/gradle/central-publishing.gradle.kts new file mode 100644 index 000000000..c108f2bb9 --- /dev/null +++ b/gradle/central-publishing.gradle.kts @@ -0,0 +1,148 @@ +// Maven Central publishing, applied by the published components (the library, the KSP processor, +// the Compose host and the Gradle plugin, which is an included build): +// +// apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) // root build +// apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) // plugin build +// +// What it adds to every Maven publication of the project (ROADMAP 15.1): +// * the POM fields Central requires: name, description, url, license, developers, scm; +// * an empty javadoc jar and, where the publication has none, an empty sources jar (Central +// refuses a jar publication without both; pom-only plugin markers need neither); +// * a local `centralStaging` repository, a plain Maven layout under /build/central-staging, +// which `.github/scripts/release/central-bundle.sh` zips for the Central Portal upload API. +// Every component writes the same directory, so one deployment carries all of them; +// * in-memory PGP signing of every publication, only when a key is given +// (MAVEN_GPG_PRIVATE_KEY / MAVEN_GPG_PASSPHRASE in the environment, or the Gradle properties +// signingInMemoryKey / signingInMemoryKeyPassword). Without a key nothing is signed, so +// `publishToMavenLocal` keeps working on a machine that has none. +// Nothing here uploads anything; the upload is .github/scripts/release/central-upload.sh. + +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven +import org.gradle.api.publish.maven.tasks.PublishToMavenLocal +import org.gradle.api.publish.maven.tasks.PublishToMavenRepository +import org.gradle.plugins.signing.Sign +import org.gradle.plugins.signing.SigningExtension + +val repoRoot: File = if (rootDir.resolve("gradle.properties").isFile && rootDir.resolve("gradle").isDirectory) { + rootDir +} else { + rootDir.parentFile +} +val stagingDir: File = (findProperty("centralStagingDir") as String?)?.let(::File) + ?: repoRoot.resolve("build/central-staging") + +val signingKey: String? = (findProperty("signingInMemoryKey") as String?) + ?: System.getenv("MAVEN_GPG_PRIVATE_KEY")?.takeIf { it.isNotBlank() } +val signingPassword: String = (findProperty("signingInMemoryKeyPassword") as String?) + ?: System.getenv("MAVEN_GPG_PASSPHRASE") + ?: "" + +val projectUrl = "https://github.com/thisisthepy/python-multiplatform" + +val emptyJar = { classifier: String -> + tasks.register("central${classifier.replaceFirstChar { it.uppercase() }}Jar") { + archiveClassifier.set(classifier) + destinationDirectory.set(layout.buildDirectory.dir("central-empty-jars")) + // Central only checks that the jar exists; the KDoc lives in the sources jar. + from(rootProject.file("LICENSE")) { into("META-INF") } + } +} + +fun publicationIsBuilt(publication: org.gradle.api.publish.Publication, logger: org.gradle.api.logging.Logger): Boolean { + publication as MavenPublication + val missing = publication.artifacts.filter { !it.file.exists() } + if (missing.isNotEmpty()) { + logger.warn("CENTRAL-SKIPPED ${publication.groupId}:${publication.artifactId}: not built here: " + + missing.joinToString { it.file.name }) + } + return missing.isEmpty() +} + +pluginManager.withPlugin("maven-publish") { + apply(plugin = "signing") + + val publishing = extensions.getByType() + publishing.repositories.maven { + name = "centralStaging" + url = stagingDir.toURI() + } + + val javadocJar = emptyJar("javadoc") + val sourcesJar = emptyJar("sources") + + publishing.publications.withType().configureEach { + val publication = this + // A plugin marker is a pom-only publication: nothing to attach. + val markerOnly = publication.name.endsWith("PluginMarkerMaven") + pom { + name.set(provider { publication.artifactId }) + description.set( + "Embeds CPython in Kotlin Multiplatform and provides Kotlin <-> Python interoperability " + + "(${publication.artifactId})." + ) + url.set(projectUrl) + licenses { + license { + name.set("The Apache License, Version 2.0") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") + } + } + developers { + developer { + id.set("thisisthepy") + name.set("thisisthepy") + email.set("thisisthepy@gmail.com") + } + } + scm { + connection.set("scm:git:https://github.com/thisisthepy/python-multiplatform.git") + developerConnection.set("scm:git:ssh://git@github.com/thisisthepy/python-multiplatform.git") + url.set(projectUrl) + } + } + if (!markerOnly) { + // Attached once every plugin has added its own artifacts (KGP adds the sources jars of + // its targets late); a classifier already present is left alone. + gradle.projectsEvaluated { + val classifiers = publication.artifacts.mapNotNull { it.classifier }.toSet() + if ("javadoc" !in classifiers) publication.artifact(javadocJar) + if ("sources" !in classifiers) publication.artifact(sourcesJar) + } + } + } + + if (signingKey != null) { + extensions.configure { + useInMemoryPgpKeys(signingKey, signingPassword) + sign(publishing.publications) + } + // KMP publications are created late and read the .asc files without declaring the tasks + // that write them; make every publish task wait for every signing task. + tasks.withType().configureEach { + dependsOn(tasks.withType()) + } + } + + // A publication whose file is not built on this machine is left out of the staging, loudly, + // rather than failing the whole deployment. Today that is only `wasmRuntime`: its zip needs a + // local Emscripten CPython build (-PwasmPythonDir) that CI does not make (ROADMAP 10.x), so + // `central-bundle.sh` lists what was left out and a release owner decides whether that is fine. + tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") } + .configureEach { + val task = this + onlyIf { publicationIsBuilt(task.publication, task.logger) } + } + // The same for mavenLocal, so `publishAllToMavenLocal` works in a checkout without the wasm build. + tasks.withType().configureEach { + val task = this + onlyIf { publicationIsBuilt(task.publication, task.logger) } + } + + // One task per component that fills the staging directory with every publication. + tasks.register("publishAllToCentralStaging") { + group = "publishing" + description = "Publishes every publication of this project to $stagingDir (signed when a key is given)." + dependsOn(tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") }) + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fe01e4a76..27581ebd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,7 +11,7 @@ androidx-espresso-core = "3.6.1" androidx-material = "1.12.0" androidx-test-junit = "1.2.1" compose-plugin = "1.11.1-ext.3" -compose-libraries = "1.11.1-ext.2" +compose-libraries = "1.11.1-ext.4" # JetBrains stopped publishing material-icons with Compose after 1.7.3 (Maven Central: newest is 1.7.3); # it is the version Compose Multiplatform 1.11.x points to, resolved against 1.11.1's ui/ui-graphics. compose-material-icons = "1.7.3" @@ -38,6 +38,15 @@ androidx-lifecycle-viewmodel = { group = "org.thisisthepy.compose.androidx.lifec androidx-lifecycle-runtime-compose = { group = "org.thisisthepy.compose.androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "compose-libraries" } kotlinx-coroutines-swing = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-browser = { group = "org.jetbrains.kotlinx", name = "kotlinx-browser", version.ref = "kotlinx-browser" } +# The fork's design systems (compose-multiplatform-core-extended `extended/design-systems`, issue #408). +designsystem = { group = "org.thisisthepy.compose.designsystem", name = "designsystem", version.ref = "compose-libraries" } +designsystem-material3 = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-material3", version.ref = "compose-libraries" } +designsystem-breeze = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-breeze", version.ref = "compose-libraries" } +designsystem-cupertino = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-cupertino", version.ref = "compose-libraries" } +designsystem-deepin = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-deepin", version.ref = "compose-libraries" } +designsystem-fluent = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-fluent", version.ref = "compose-libraries" } +designsystem-gnome = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-gnome", version.ref = "compose-libraries" } +designsystem-liquid-glass = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-liquid-glass", version.ref = "compose-libraries" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/python-checksums.properties b/python-checksums.properties index 806f0bf69..956b5db07 100644 --- a/python-checksums.properties +++ b/python-checksums.properties @@ -1,10 +1,7 @@ # Python Multiplatform Checksums -android-aarch64-3.14.7=6d50cc3aa66e414a439594089bcdfb5f1264358155c70c1f00471c24cfb477fb -android-aarch64-3.15.0=571e4cf4e0aee3d1ab54a52ca3d4e6845b85ec0bb327f7debfb873172f445dab -android-x86_64-3.14.7=2c16ce2359565cd8c24f86cfb75630768ba6607e732946b294b969797f583b60 -android-x86_64-3.15.0=2413d51948d612cf42baefba25295bb83c2ed84df5726af6fe5bde2dec5d87e7 -ios-3.14-b11=b591f3301bd22a4f423c49c746cac9e55558b909fd14d6eb8327ccc62234ab7b -ios-3.15.0-pythonorg=68e98ab35cc0643ba3a227a8c7768da45c2a5e67dbe1bc26598537e96680fd19 +android-aarch64-3.15.0-freethreaded=40995c530c3e454d829b4fdea8404d8c0b0e58f60c2d41d400daa1fb0ab65991 +android-x86_64-3.15.0-freethreaded=d52580653858c429f542db0ca4ecfe12452faf6c9a967bb4b62797df7c2c8d24 +ios-3.15.0-freethreaded=b8c0a4b32d31738f6ead7ace28acee0cb2a590bd0ca7015e9c32af613445e6b5 linux-x86_64-3.14.7-20260807=a2478d654ed51d443bae21ec20ad927f116b4f5aae4094ab74918a6aa38f0575 linux-x86_64-3.14.7-20260807-freethreaded=13f80b7fc242814c0be4e2a4d8283f39c7e1c0f3f0d8f6d6a15cedb5b62eb6d3 linux-x86_64-3.15.0-20261013=69740dcd9b3480b3d53760f2d3b75582adb14fa54434628dea32480ddede542a diff --git a/python-multiplatform-compose/build.gradle.kts b/python-multiplatform-compose/build.gradle.kts index 011f95798..11cb6e6fc 100644 --- a/python-multiplatform-compose/build.gradle.kts +++ b/python-multiplatform-compose/build.gradle.kts @@ -7,18 +7,22 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget * composable. * * A separate module so that `:python-multiplatform` itself never depends on Compose (same naming as - * `-ksp` and `-gradle-plugin`). JVM targets first -- desktop and Android share the code, which is all - * in `commonMain`; iOS and wasm come later. + * `-ksp` and `-gradle-plugin`). Desktop JVM, Android and Kotlin/Native desktop (macosArm64, linuxX64) + * share the code, which is all in `commonMain`; iOS and wasm come later. */ plugins { alias(libs.plugins.kotlin.multiplatform) alias(libs.plugins.android.library) alias(libs.plugins.jetpack.compose) alias(libs.plugins.compose.compiler) + id("maven-publish") } kotlin { androidTarget { + // Without a named variant KMP creates no Android publication (ROADMAP 15d, same as the library). + publishLibraryVariants("release") + @OptIn(ExperimentalKotlinGradlePluginApi::class) compilerOptions { jvmTarget.set(JvmTarget.JVM_17) @@ -31,6 +35,12 @@ kotlin { } } + // Kotlin/Native desktop (#191 Phase B, #386), the targets `:python-multiplatform` has since #370. + // Upstream JetBrains Compose publishes no `ui` klib for linuxX64; the fork this repository builds + // on publishes `runtime`, `runtime-saveable` and `ui` for both targets. + macosArm64() + linuxX64() + sourceSets { commonMain.dependencies { api(projects.pythonMultiplatform) @@ -38,6 +48,13 @@ kotlin { api(compose.ui) api(compose.runtimeSaveable) } + // Same shape as the library's: commonMain -> desktopNativeMain -> {macosArm64Main, linuxX64Main}. + // Empty today (the host is all commonMain); it is where K/N-desktop-only host code goes. + val desktopNativeMain by creating { + dependsOn(commonMain.get()) + } + getByName("macosArm64Main").dependsOn(desktopNativeMain) + getByName("linuxX64Main").dependsOn(desktopNativeMain) } } @@ -52,3 +69,8 @@ android { targetCompatibility = JavaVersion.VERSION_17 } } + +// Published to Maven Central beside the library as org.thisisthepy:python-multiplatform-compose, at the +// root's one version (#70): POM fields, sources/javadoc jars, centralStaging repository and in-memory +// signing come from the shared script (ROADMAP 15.1). +apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) diff --git a/python-multiplatform-compose/fixtures/compose/build.gradle.kts b/python-multiplatform-compose/fixtures/compose/build.gradle.kts index 40470cf26..64f770a11 100644 --- a/python-multiplatform-compose/fixtures/compose/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/compose/build.gradle.kts @@ -58,6 +58,17 @@ kotlin { exclude(group = "org.jetbrains.compose.runtime") exclude(group = "org.jetbrains.compose.ui") } + // The fork's design systems (issue #408): `pythonx.compose.material3` puts + // `org.thisisthepy.compose.material3` in front of `androidx.compose.material3`, and + // `pythonx.compose.` is each of these (pythonx-compose `pythonx-map.toml`). + implementation(libs.designsystem) + implementation(libs.designsystem.material3) + implementation(libs.designsystem.breeze) + implementation(libs.designsystem.cupertino) + implementation(libs.designsystem.deepin) + implementation(libs.designsystem.fluent) + implementation(libs.designsystem.gnome) + implementation(libs.designsystem.liquid.glass) implementation(compose.desktop.currentOs) } } @@ -123,6 +134,15 @@ pythonBindings { // Python calls inside a composition. Its file facade, not the package, so that the host // composables (`PythonAppView` and friends) stay out of the table. "python.multiplatform.compose.RememberSaveableKt", + // 8. The fork's design systems (issue #408), one package each. + "org.thisisthepy.compose.designsystem", + "org.thisisthepy.compose.material3", + "org.thisisthepy.compose.breeze", + "org.thisisthepy.compose.cupertino", + "org.thisisthepy.compose.deepin", + "org.thisisthepy.compose.fluent", + "org.thisisthepy.compose.gnome", + "org.thisisthepy.compose.liquidglass", ), ) // Stubs on: `generatePythonStubs` over the real Compose jars is what `.github/scripts/stubs/check-stubs.sh` (the mypy diff --git a/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt b/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt new file mode 100644 index 000000000..39c56f4df --- /dev/null +++ b/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt @@ -0,0 +1,80 @@ +package fixture.compose + +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.pythonx.PythonxAdapter +import python.multiplatform.ffi.upcall.UpcallBootstrap +import python.multiplatform.generated.artifacts.ArtifactTable +import python.multiplatform.reflection.UpcallTable +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test + +/** + * SPEC B-12 (issue #408): the fork's design systems (`org.thisisthepy.compose.designsystem:*`) are + * walked, each under its own Kotlin package, so pythonx-compose's `pythonx.compose.material3` (the + * design system in front of `androidx.compose.material3`) and `pythonx.compose.` resolve. + * + * Each package must answer with the declaration that names the system; `materialDesignSystem` and + * `MaterialButton` are checked parameter by parameter, by their Kotlin names, because pythonx-compose + * calls them. Without the dependency or the walked package, the import fails. + */ +class DesignSystemBindingTest { + + @BeforeTest + fun installProducers() { + Python3.initialize(silent = true) + UpcallTable.clear() + UpcallTable.install(ArtifactTable.fragments) + check(UpcallBootstrap.publishToGlobals()) { "UpcallBootstrap.publishToGlobals() failed" } + PythonxAdapter.install(PYTHONX_RAW_VALUE_CLASSES) + } + + @AfterTest + fun cleanup() { + UpcallTable.clear() + } + + @Test + fun everyDesignSystemPackageIsBoundUnderItsKotlinName() { + Python3.exec( + """ + import importlib as _ds_il + _ds_expected = { + 'org.thisisthepy.compose.designsystem': 'LocalDesignSystem', + 'org.thisisthepy.compose.material3': 'Material3DesignSystem', + 'org.thisisthepy.compose.breeze': 'BreezeDesignSystem', + 'org.thisisthepy.compose.cupertino': 'CupertinoDesignSystem', + 'org.thisisthepy.compose.deepin': 'DeepinDesignSystem', + 'org.thisisthepy.compose.fluent': 'FluentDesignSystem', + 'org.thisisthepy.compose.gnome': 'GnomeDesignSystem', + 'org.thisisthepy.compose.liquidglass': 'LiquidGlass', + } + _ds_missing = {} + for _ds_package, _ds_name in _ds_expected.items(): + _ds_module = _ds_il.import_module(_ds_package) + if _ds_name not in dir(_ds_module): + _ds_missing[_ds_package] = sorted(n for n in dir(_ds_module) if not n.startswith('_'))[:20] + assert not _ds_missing, _ds_missing + """.trimIndent(), + ) + } + + @Test + fun theMaterialDesignSystemFunctionsCarryTheirKotlinParameters() { + Python3.exec( + """ + import python_multiplatform as _ds_pm + import org.thisisthepy.compose.material3 as _ds_m3 + + _ds_d = _ds_pm.describe(_ds_m3, 'materialDesignSystem') + assert [(_p['name'], _p['type']) for _p in _ds_d['parameters']] == \ + [('isDark', 'kotlin.Boolean')], _ds_d + + _ds_b = _ds_pm.describe(_ds_m3, 'MaterialButton') + assert [_p['name'] for _p in _ds_b['parameters']] == \ + ['variant', 'onClick', 'modifier', 'enabled', 'content'], _ds_b + assert _ds_b['parameters'][0]['type'] == 'org.thisisthepy.compose.designsystem.ButtonVariant', _ds_b + """.trimIndent(), + ) + } +} diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/README.md b/python-multiplatform-compose/fixtures/notebook-e2e/README.md index af55ae435..ef0399bf7 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/README.md +++ b/python-multiplatform-compose/fixtures/notebook-e2e/README.md @@ -68,11 +68,12 @@ JAVA_HOME= ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e `site-packages/` (the wheel) and `app/`. The image finds them there with no environment. - `notebookE2eNativeRun` removes `JAVA_HOME` and `PYTHONHOME`, requires 20 passed, and fails if `otool -L` shows `libjvm`. -- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended's mavenLocal - build (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread +- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended on Maven + Central (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread work to skiko's Swing queue, which needs AWT, and AWT needs `libjvm`. The fork sends it to `Dispatchers.Main` under `compose.main.dispatcher=coroutines`, which `NotebookE2eMainDispatcherFactory` supplies (registered by `src/desktopTest/native-image-main/` on the image's classpath only). + CI runs `notebookE2eNativeAgent` and `notebookE2eNativeRun` in `.github/workflows/desktop.yml` (`notebook-e2e-native-image`, #251). ## Scenarios diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts index 8c4915a86..3b17746a6 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts @@ -61,6 +61,17 @@ kotlin { exclude(group = "org.jetbrains.compose.runtime") exclude(group = "org.jetbrains.compose.ui") } + // The fork's design systems (issue #408): `pythonx.compose.material3` puts + // `org.thisisthepy.compose.material3` in front of `androidx.compose.material3`, and + // `pythonx.compose.` is each of these (pythonx-compose `pythonx-map.toml`). + implementation(libs.designsystem) + implementation(libs.designsystem.material3) + implementation(libs.designsystem.breeze) + implementation(libs.designsystem.cupertino) + implementation(libs.designsystem.deepin) + implementation(libs.designsystem.fluent) + implementation(libs.designsystem.gnome) + implementation(libs.designsystem.liquid.glass) implementation(compose.desktop.currentOs) } } @@ -98,6 +109,15 @@ pythonBindings { "androidx.compose.foundation.text.input.TextFieldState", "androidx.compose.foundation.text.input.TextFieldStateKt", "python.multiplatform.compose.RememberSaveableKt", + // The fork's design systems (issue #408), one package each. + "org.thisisthepy.compose.designsystem", + "org.thisisthepy.compose.material3", + "org.thisisthepy.compose.breeze", + "org.thisisthepy.compose.cupertino", + "org.thisisthepy.compose.deepin", + "org.thisisthepy.compose.fluent", + "org.thisisthepy.compose.gnome", + "org.thisisthepy.compose.liquidglass", ), ) } @@ -280,7 +300,10 @@ val e2eMainClass = "fixture.notebook.NotebookE2eMainKt" val e2eNativeConfig = layout.projectDirectory.dir("src/desktopTest/native-image") val e2eNativeDir = layout.buildDirectory.dir("native/notebook-e2e") val e2eAppDir = layout.projectDirectory.dir("src/desktopTest/python") -val e2eJdk25 = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) } +val e2eJdk25 = javaToolchains.launcherFor { + languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) +} fun JavaExec.notebookE2eOnTheJvm() { dependsOn(e2eTestCompilation.compileTaskProvider, installPythonxComposeWheel) diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index 384ce7ce4..70f38410c 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -17,7 +17,7 @@ val rootGradleProperties = Properties().apply { } fun rootProperty(name: String, fallback: String): String = findProperty(name)?.toString() ?: rootGradleProperties.getProperty(name) ?: fallback -val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha02")}" +val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha03")}" version = publishedVersion /** @@ -251,7 +251,10 @@ val generateCoordinates = tasks.register("generateCoordinates") { // case where this build is checked out without the root's properties beside it. val pythonVersionValue = rootProperty("pythonVersion", "3.15.0") val pbsReleaseValue = rootProperty("pythonBuildStandaloneRelease", "20261013") - val freeThreadedValue = rootProperty("pythonFreeThreaded", "false").toBoolean() + // Desktop embeds the free-threaded build only (issue #158, maintainer 2026-10-04: "Free-threaded + // is the default and the only build. The GIL build is not kept as an option."), so there is no + // root property to read any more: the library's `desktopJar` always carries `libpython3.15t`. + val freeThreadedValue = true // The library and this plugin publish under one version since #70, so the wasm-runtime zip // (published beside the library artifact) is at this build's own version. @@ -260,33 +263,20 @@ val generateCoordinates = tasks.register("generateCoordinates") { val klibWorkerValue = klibWorkerCoordinates - // The CPython iOS support archive the library build downloads, derived with the same rule as - // `python-multiplatform/build.gradle.kts` (`downloadPython_ios`): python.org from 3.15, BeeWare - // before. The SHA-256 is read from the root `python-checksums.properties` under the library's - // own lock key and baked into the plugin, so a consumer verifies against the pin this plugin - // was built with and needs no lockfile of its own. - val iosVersionParts = pythonVersionValue.split(".") - val iosFromPythonOrgValue = iosVersionParts[0].toInt() > 3 || (iosVersionParts[0].toInt() == 3 && iosVersionParts[1].toInt() >= 15) - val iosLibVersion = "${iosVersionParts[0]}.${iosVersionParts[1]}" - val iosReleaseDir = "$iosLibVersion.${iosVersionParts.getOrNull(2)?.takeWhile { it.isDigit() } ?: "0"}" - val appleSupportBuild = rootProperty("pythonAppleSupportBuild", "b11") - val iosArchiveNameValue = if (iosFromPythonOrgValue) { - "python-$pythonVersionValue-iOS-XCframework.tar.gz" - } else { - "Python-$iosLibVersion-iOS-support.$appleSupportBuild.tar.gz" - } - val iosUrlValue = if (iosFromPythonOrgValue) { - "https://www.python.org/ftp/python/$iosReleaseDir/$iosArchiveNameValue" - } else { - "https://github.com/beeware/Python-Apple-support/releases/download/$iosLibVersion-$appleSupportBuild/$iosArchiveNameValue" - } - val iosLockKeyValue = if (iosFromPythonOrgValue) "ios-$pythonVersionValue-pythonorg" else "ios-$iosLibVersion-$appleSupportBuild" + // The iOS support archive is the project's own free-threaded 3.15t build (#163, #328; owner + // decision #158: free-threaded 3.15t is the only build, no GIL archive is kept). Release + // `cpython-t-mobile` of thisisthepy/python-multiplatform, pinned under + // `ios--freethreaded` in the root `python-checksums.properties` and baked into the plugin, + // so a consumer verifies against the pin this plugin was built with and needs no lockfile of its + // own. The 3.15.0 value is the release's SHA256SUMS line, equal to the GitHub asset digest and to + // `shasum -a 256` of the downloaded archive (checked 2026-11-13). val checksumsFile = project.rootDir.resolve("../python-checksums.properties") - val iosSha256Value = Properties().apply { + val iosFreeThreadedLockKeyValue = "ios-$pythonVersionValue-freethreaded" + val iosFreeThreadedSha256Value = Properties().apply { if (checksumsFile.isFile) checksumsFile.inputStream().use { load(it) } - }.getProperty(iosLockKeyValue) + }.getProperty(iosFreeThreadedLockKeyValue) ?: throw GradleException( - "python-checksums.properties (${checksumsFile.path}) has no '$iosLockKeyValue'; the plugin bakes the iOS support archive's SHA-256 from it.", + "python-checksums.properties (${checksumsFile.path}) has no '$iosFreeThreadedLockKeyValue'; the plugin bakes the iOS support archive's SHA-256 from it.", ) inputs.property("coordinates", coordinates) @@ -295,10 +285,8 @@ val generateCoordinates = tasks.register("generateCoordinates") { inputs.property("pbsRelease", pbsReleaseValue) inputs.property("pythonFreeThreaded", freeThreadedValue) inputs.property("wasmRuntimeCoordinates", wasmRuntimeCoordinatesValue) - inputs.property("iosUrl", iosUrlValue) - inputs.property("iosArchiveName", iosArchiveNameValue) - inputs.property("iosLockKey", iosLockKeyValue) - inputs.property("iosSha256", iosSha256Value) + inputs.property("iosFreeThreadedLockKey", iosFreeThreadedLockKeyValue) + inputs.property("iosFreeThreadedSha256", iosFreeThreadedSha256Value) outputs.dir(outputDir) doLast { val file = outputDir.get().file("python/multiplatform/gradle/ProcessorCoordinates.kt").asFile @@ -316,17 +304,11 @@ val generateCoordinates = tasks.register("generateCoordinates") { /** The python-build-standalone release tag that published it. */ internal const val DEFAULT_PBS_RELEASE: String = "$pbsReleaseValue" - /** The pinned CPython iOS support archive: where it is downloaded from. */ - internal const val DEFAULT_IOS_ARCHIVE_URL: String = "$iosUrlValue" - - /** Its file name, which is also the file's name under the consumer's `build/python-standalone/`. */ - internal const val DEFAULT_IOS_ARCHIVE_NAME: String = "$iosArchiveNameValue" - - /** The key it has in the root `python-checksums.properties`, kept for messages. */ - internal const val DEFAULT_IOS_LOCK_KEY: String = "$iosLockKeyValue" + /** The key of the iOS support archive (the 3.15t build, SPEC T-2a) in `python-checksums.properties`. */ + internal const val DEFAULT_IOS_FREE_THREADED_LOCK_KEY: String = "$iosFreeThreadedLockKeyValue" /** Its SHA-256, read from `python-checksums.properties` when this plugin was built. */ - internal const val DEFAULT_IOS_ARCHIVE_SHA256: String = "$iosSha256Value" + internal const val DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256: String = "$iosFreeThreadedSha256Value" internal const val DEFAULT_PYTHON_FREE_THREADED: Boolean = $freeThreadedValue @@ -418,3 +400,5 @@ tasks.test { inputs.files(walkerFixtureKlib).withPropertyName("walkerFixtureKlib").withPathSensitivity(PathSensitivity.NAME_ONLY) systemProperty("python.multiplatform.walkerFixtureKlib", walkerFixtureKlib.singleFile.absolutePath) } + +apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) diff --git a/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts b/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts index a5d1c44db..205f49cf5 100644 --- a/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts @@ -52,9 +52,10 @@ kotlin { // Mirrors `:python-multiplatform-ksp:fixtures:app`'s own androidNativeArm64 linker config: the test binary needs // the same CPython symbols resolved at final link time, since `python-multiplatform`'s klib // only declares them via cinterop and does not itself embed a static libpython. - val targetExtractDir = "$pythonDownloadDir/extracted/$fixturePythonVersion/android-aarch64/prefix" + // The free-threaded 3.15t tree, the only Android build (#328). + val targetExtractDir = "$pythonDownloadDir/extracted/$fixturePythonVersion/android-aarch64-freethreaded/prefix" binaries.getTest(org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG).linkerOpts.addAll( - listOf("-L$targetExtractDir/lib/", "-lpython$fixturePythonLibVersion", "-Wl,--allow-shlib-undefined"), + listOf("-L$targetExtractDir/lib/", "-lpython${fixturePythonLibVersion}t", "-Wl,--allow-shlib-undefined"), ) } // Issue #234: the simulator target, so the suite runs through Gradle (`iosSimulatorArm64Test`) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt index a95951aa5..5f438c990 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt @@ -14,9 +14,11 @@ object AcquiredHeaderVersion { requireNotNull(pyVersion.find(patchlevelText)) { "no PY_VERSION in patchlevel.h" }.groupValues[1] fun patchlevelFile(extractedRoot: File, target: String, pythonVersion: String): File = - extractedRoot.resolve(CPythonIncludeLayout.relativePath(target, CPythonFlavour.GIL, pythonVersion)).resolve("patchlevel.h") + extractedRoot.resolve( + CPythonIncludeLayout.relativePath(target, CPythonIncludeLayout.embeddedFlavour(target), pythonVersion), + ).resolve("patchlevel.h") - /** target -> description, for every GIL target whose headers exist and report a different version. */ + /** target -> description, for every target whose embedded headers exist and report a different version. */ fun mismatches(extractedRoot: File, pythonVersion: String): Map = CPythonIncludeLayout.allTargets.mapNotNull { t -> val f = patchlevelFile(extractedRoot, t, pythonVersion) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt index 5b4862399..36b0beb1d 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt @@ -28,8 +28,21 @@ object CPythonIncludeLayout { val allTargets: List = desktopTargets + androidTargets + iosTargets + /** + * The flavour `python-multiplatform` embeds for [target], which is the only one its build + * extracts: free-threaded on every target (issue #158, maintainer 2026-10-04: "Free-threaded is + * the default and the only build"; desktop since #398, Android and iOS since #328). + */ + @Suppress("UNUSED_PARAMETER") + fun embeddedFlavour(target: String): CPythonFlavour = CPythonFlavour.FREE_THREADED + + /** + * Whether a layout exists for [target] in [flavour]. Free-threaded exists everywhere. A GIL + * layout is kept for the desktop targets only (python-build-standalone publishes both, and the + * paths differ); Android and iOS have no GIL build to lay out since #328. + */ fun supports(target: String, flavour: CPythonFlavour): Boolean = - target in allTargets && (flavour == CPythonFlavour.GIL || target in desktopTargets) + target in allTargets && (flavour == CPythonFlavour.FREE_THREADED || target in desktopTargets) /** The extraction task that produces [target] -- the name `python-multiplatform/build.gradle.kts` registers. */ fun downloadTaskName(target: String): String = when (target) { @@ -52,9 +65,11 @@ object CPythonIncludeLayout { val dir = if (flavour == CPythonFlavour.FREE_THREADED) "$target-freethreaded" else target "$dir/python/include/python$tag" } - in androidTargets -> "$target/prefix/include/python$tag" + // `-freethreaded` in the directory name, as on desktop: a GIL `android-aarch64/` tree from + // an older checkout must never be mistaken for the 3.15t one (#328). + in androidTargets -> "$target-freethreaded/prefix/include/python$tag" // The framework's own Headers, which is also what the library's iOS cinterop reads: the - // 3.15 iOS support archive no longer ships `/include/python3.15` beside it (#313). + // iOS support archive ships no `/include/python3.15t` beside it (#313). else -> "ios/Python.xcframework/$target/Python.framework/Headers" } } @@ -64,9 +79,12 @@ object CPythonIncludeLayout { /** Android links `libpython3.x.so`, Windows links the import library; macOS/Linux/iOS need none. */ fun isLinkRequired(target: String): Boolean = target in androidTargets || target == "windows-x86_64" - /** Free-threaded link libraries exist on Windows only (`python314t.lib`); Android has no free-threaded build. */ + /** + * Android links `libpython3.15t.so` (free-threaded only since #328); Windows has both + * `python315.lib` and `python315t.lib`. + */ fun supportsLinkLibrary(target: String, flavour: CPythonFlavour): Boolean = - isLinkRequired(target) && (flavour == CPythonFlavour.GIL || target == "windows-x86_64") + isLinkRequired(target) && (flavour == CPythonFlavour.FREE_THREADED || target == "windows-x86_64") /** Directory holding the link library, relative to the version-keyed extraction root. */ fun linkLibraryDirRelativePath(target: String, flavour: CPythonFlavour): String { @@ -75,11 +93,11 @@ object CPythonIncludeLayout { "windows-x86_64" -> (if (flavour == CPythonFlavour.FREE_THREADED) "windows-x86_64-freethreaded" else "windows-x86_64") + "/python/libs" - else -> "$target/prefix/lib" + else -> "$target-freethreaded/prefix/lib" } } - /** File name of the link library, e.g. `libpython3.14.so` or `python314.lib` (`python314t.lib` free-threaded). */ + /** File name of the link library, e.g. `libpython3.15t.so` or `python315.lib` (`python315t.lib` free-threaded). */ fun linkLibraryFileName(target: String, flavour: CPythonFlavour, pythonVersion: String): String { require(supportsLinkLibrary(target, flavour)) { "no $flavour link library for target '$target'" } val parts = pythonVersion.split('.') @@ -87,7 +105,7 @@ object CPythonIncludeLayout { return if (target == "windows-x86_64") { "python${parts[0]}${parts[1]}" + (if (flavour == CPythonFlavour.FREE_THREADED) "t" else "") + ".lib" } else { - "libpython${parts[0]}.${parts[1]}.so" + "libpython${parts[0]}.${parts[1]}t.so" } } } @@ -97,9 +115,9 @@ object CPythonIncludeLayout { * task as its build dependency, so anything that wires it as an input (`-I`, an `@InputDirectory`) * triggers the download and unpack. * - * Only the flavour the build was configured for (`-PpythonFreeThreaded`) has an extraction task; - * asking for the other one fails with a message naming the property, rather than returning a path - * nothing will ever create. + * Only the flavour the library embeds ([CPythonIncludeLayout.embeddedFlavour]: free-threaded on + * every target since #328) has an extraction task, and it is the default `flavour`; asking for the + * other one fails loudly rather than returning a path nothing will ever create. * * @param extractedRoot the version-keyed extraction directory * @param extractionTask the task that fills [extractedRoot] for a target+flavour, or null when this build does not extract it @@ -112,20 +130,20 @@ class CPythonIncludeDirectories( val targets: List get() = CPythonIncludeLayout.allTargets /** Whether [includeDir] would succeed for this pair in this build. */ - fun isAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean = + fun isAvailable(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Boolean = CPythonIncludeLayout.supports(target, flavour) && extractionTask(target, flavour) != null /** The directory holding `Python.h` for [target]; resolving it makes Gradle run the extraction task first. */ - fun includeDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + fun includeDir(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { if (!CPythonIncludeLayout.supports(target, flavour)) { throw GradleException( "No $flavour CPython include directory for '$target'. Targets: ${CPythonIncludeLayout.allTargets}; " + - "free-threaded exists for desktop targets only.", + "Android and iOS have no GIL build (#328).", ) } val task = extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour selected with -PpythonFreeThreaded is downloaded.", + "Only the flavour the library embeds is downloaded: free-threaded on every target (#158, #328).", ) val relative = CPythonIncludeLayout.relativePath(target, flavour, pythonVersion) // `TaskProvider.map` keeps the task as the producer, so the dependency travels with the value. @@ -136,31 +154,31 @@ class CPythonIncludeDirectories( fun isLinkRequired(target: String): Boolean = CPythonIncludeLayout.isLinkRequired(target) /** Whether [libraryDir]/[libraryFile] would succeed for this pair in this build. */ - fun isLinkAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean = + fun isLinkAvailable(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Boolean = CPythonIncludeLayout.supportsLinkLibrary(target, flavour) && extractionTask(target, flavour) != null private fun linkTask(target: String, flavour: CPythonFlavour): TaskProvider<*> { if (!CPythonIncludeLayout.supportsLinkLibrary(target, flavour)) { throw GradleException( "No $flavour CPython link library for '$target'. Only android-* and windows-x86_64 link against " + - "CPython; macOS, Linux and iOS need none (isLinkRequired=false); free-threaded exists on Windows only.", + "CPython; macOS, Linux and iOS need none (isLinkRequired=false); Android has no GIL build (#328).", ) } return extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour selected with -PpythonFreeThreaded is downloaded.", + "Only the flavour the library embeds is downloaded: free-threaded on every target (#158, #328).", ) } /** The directory holding the link library (the `-L` directory); resolving it runs the extraction task first. */ - fun libraryDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + fun libraryDir(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { val task = linkTask(target, flavour) val relative = CPythonIncludeLayout.linkLibraryDirRelativePath(target, flavour) return task.map { extractedRoot.dir(relative) } } - /** The link library file itself (`libpython3.14.so` / `python314.lib`); carries the extraction task. */ - fun libraryFile(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + /** The link library file itself (`libpython3.15t.so` / `python315t.lib`); carries the extraction task. */ + fun libraryFile(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { val task = linkTask(target, flavour) val relative = CPythonIncludeLayout.linkLibraryDirRelativePath(target, flavour) + "/" + CPythonIncludeLayout.linkLibraryFileName(target, flavour, pythonVersion) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt index 2b4a714fe..11033e031 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt @@ -10,6 +10,11 @@ import org.gradle.api.Project * - as the Gradle attributes [ATTRIBUTE_VERSION] / [ATTRIBUTE_FREE_THREADED] on every consumable * `*Elements` configuration (published Gradle module metadata), * - as the classpath resource [RESOURCE_PATH] (read from the jar/AAR, no Gradle needed). + * + * One instance describes one build of CPython. A project whose platforms embed different builds + * (issues #158, #328: desktop, Android and iOS 3.15t, wasm a GIL build until #175) passes a per-variant + * `freeThreadedFor` to [publishAttributes] / [register], so each published variant states the build + * it really carries, and renders one resource per artefact. */ class EmbeddedPythonVersion(val pythonVersion: String, val freeThreaded: Boolean) { @@ -23,22 +28,30 @@ class EmbeddedPythonVersion(val pythonVersion: String, val freeThreaded: Boolean fun renderProperties(): String = "pythonVersion=$pythonVersion\nfreeThreaded=$freeThreaded\n" - /** Adds the two attributes to every consumable `*Elements` configuration, including ones created later. */ - fun publishAttributes(project: Project) { + /** + * Adds the attributes to every consumable `*Elements` configuration, including ones created later. + * + * [ATTRIBUTE_VERSION] is [pythonVersion] everywhere. [ATTRIBUTE_FREE_THREADED] is + * `freeThreadedFor(configurationName)`: by default [freeThreaded] for every variant; a + * multi-platform project passes a function that answers per variant, and `null` leaves the + * attribute off a variant that carries no CPython of its own (shared metadata). + */ + fun publishAttributes(project: Project, freeThreadedFor: (configurationName: String) -> Boolean? = { freeThreaded }) { project.configurations .matching { it.isCanBeConsumed && it.name.endsWith("Elements") } .configureEach { attributes.attribute(org.gradle.api.attributes.Attribute.of(ATTRIBUTE_VERSION, String::class.java), pythonVersion) + val ft = freeThreadedFor(name) ?: return@configureEach attributes.attribute( org.gradle.api.attributes.Attribute.of(ATTRIBUTE_FREE_THREADED, String::class.java), - freeThreaded.toString(), + ft.toString(), ) } } - fun register(project: Project) { + fun register(project: Project, freeThreadedFor: (configurationName: String) -> Boolean? = { freeThreaded }) { project.extensions.add(EXTENSION_NAME, this) - publishAttributes(project) + publishAttributes(project, freeThreadedFor) } companion object { diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt index ae7f178f6..cfb17b23e 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt @@ -35,6 +35,42 @@ interface PythonBindingsIosExtension { val linkPythonFramework: Property } +/** + * The project's own free-threaded CPython builds for Android and iOS (#163, `build-ft-mobile.yml`), + * published on the GitHub release `cpython-t-mobile` of `thisisthepy/python-multiplatform` + * because neither python.org nor BeeWare publishes a free-threaded mobile build. Names only; the + * SHA-256 pins live in the root `python-checksums.properties` under [lockKey]. + */ +internal object FreeThreadedMobileRelease { + const val REPOSITORY: String = "thisisthepy/python-multiplatform" + + fun tag(pythonVersion: String): String = "cpython-${pythonVersion}t-mobile" + + fun iosArchiveName(pythonVersion: String): String = "python-${pythonVersion}t-iOS-XCframework.tar.gz" + + /** `aarch64` and `x86_64` (#396; the owner approved the x86_64 3.15t build on 2026-11-13). The library build reads them. */ + fun androidArchiveName(pythonVersion: String, arch: String): String = "python-${pythonVersion}t-$arch-linux-android.tar.gz" + + fun url(pythonVersion: String, assetName: String): String = + "https://github.com/$REPOSITORY/releases/download/${tag(pythonVersion)}/$assetName" + + /** `ios-3.15.0-freethreaded`, `android-aarch64-3.15.0-freethreaded`. */ + fun lockKey(platform: String, pythonVersion: String): String = "$platform-$pythonVersion-freethreaded" +} + +/** One iOS support archive the consumer wiring can download: where from, under which name, which pin. */ +internal data class IosArchive(val url: String, val name: String, val sha256: String) + +/** + * The archive [configureIosConsumer] downloads: the project's own free-threaded `3.15t` build of + * [DEFAULT_PYTHON_VERSION]. It is the only iOS build (owner decision #158, SPEC T-2a); there is no + * GIL archive and no switch. + */ +internal fun iosArchive(): IosArchive { + val name = FreeThreadedMobileRelease.iosArchiveName(DEFAULT_PYTHON_VERSION) + return IosArchive(FreeThreadedMobileRelease.url(DEFAULT_PYTHON_VERSION, name), name, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) +} + internal const val DOWNLOAD_IOS_TASK = "downloadPython_ios" internal const val PREPARE_XCFRAMEWORK_TASK = "preparePythonXcframework" internal const val STAGE_IOS_HOME_TASK = "stageIosPythonHome" @@ -106,6 +142,13 @@ abstract class DownloadPythonIosTask : DefaultTask() { work.mkdirs() val archive = File(work, archiveName.get()) val expected = expectedSha256.get().lowercase() + if (expected.isBlank()) { + throw GradleException( + "There is no pinned SHA-256 for ${archiveName.get()}: the root python-checksums.properties had no " + + "entry for it ('$DEFAULT_IOS_FREE_THREADED_LOCK_KEY') when this plugin was built. " + + "Nothing was downloaded.", + ) + } if (archive.isFile && sha256Hex(archive) != expected) { logger.lifecycle("The cached ${archive.name} does not match its pinned SHA-256; deleting it and downloading once more.") @@ -202,13 +245,14 @@ internal fun PythonBindingsPlugin.configureIosConsumer(project: Project, extensi if (iosTargets.isEmpty()) return val build = project.layout.buildDirectory + val archive = iosArchive() val download = project.tasks.register(DOWNLOAD_IOS_TASK, DownloadPythonIosTask::class.java) { group = "python" - description = "Downloads and verifies the pinned CPython iOS support archive into build/python-standalone (issue #90)" - archiveUrl.convention(DEFAULT_IOS_ARCHIVE_URL) - archiveName.convention(DEFAULT_IOS_ARCHIVE_NAME) - expectedSha256.convention(DEFAULT_IOS_ARCHIVE_SHA256) + description = "Downloads and verifies the pinned free-threaded CPython 3.15t iOS support archive into build/python-standalone (issue #90)" + archiveUrl.convention(archive.url) + archiveName.convention(archive.name) + expectedSha256.convention(archive.sha256) workDirectory.convention(build.dir("python-standalone")) } val xcframeworkSource = download.flatMap { task -> task.workDirectory.dir("extracted/ios/Python.xcframework") } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt index 5fef66032..572e0a069 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt @@ -23,8 +23,8 @@ import java.nio.file.StandardCopyOption * * ``` * Python.xcframework/ - * lib/python3.14/ pure-Python stdlib, shared by every slice - * ios-arm64/lib-arm64/python3.14/ lib-dynload/ (the .so files), _sysconfigdata (device) + * lib/python3.15t/ pure-Python stdlib, shared by every slice + * ios-arm64/lib-arm64/python3.15t/ lib-dynload/ (the .so files), _sysconfigdata (device) * ios-arm64_x86_64-simulator/lib-arm64/... the same for the arm64 simulator * ios-arm64_x86_64-simulator/lib-x86_64/... ...and the x86_64 simulator * build/iOS-dylib-Info-template.plist Info.plist for an extension-module framework @@ -116,15 +116,18 @@ object IosPythonHomeLayout { ) } - /** `3.14` from `3.14.7`. */ + /** + * `3.15t` from `3.15.0`: the iOS framework is always free-threaded (SPEC T-2a), so its standard + * library is `lib/python3.15t/` and its slices carry `lib-/python3.15t/`. + */ fun stdlibTag(pythonVersion: String): String { val parts = pythonVersion.split('.') require(parts.size >= 2) { "python version '$pythonVersion' has no major.minor" } - return "${parts[0]}.${parts[1]}" + return "${parts[0]}.${parts[1]}t" } /** - * The trees merged into `lib/python/` for [slice], relative to the xcframework, shared tree + * The trees merged into `lib/pythont/` for [slice], relative to the xcframework, shared tree * first: the slice tree only adds files (`lib-dynload/`, `_sysconfigdata__ios_*`), but if it ever * overlapped, the slice-specific file is the one that must win, as in `install_stdlib`. */ @@ -134,7 +137,7 @@ object IosPythonHomeLayout { } /** - * Whether a file or directory at [stdlibRelativePath] -- relative to `lib/python/`, `/` + * Whether a file or directory at [stdlibRelativePath] -- relative to `lib/pythont/`, `/` * separated -- goes into the app. * * Not copied: any `__pycache__` (bytecode from whatever ran against the extracted tree, so the @@ -183,7 +186,7 @@ object IosPythonHomeLayout { } /** - * Stages one slice's prefix for an iOS app bundle (SPEC L-11): `/home/lib/python/` + * Stages one slice's prefix for an iOS app bundle (SPEC L-11): `/home/lib/pythont/` * as [IosPythonHomeLayout.plan] lists it, plus `/dylib-Info-template.plist` *beside* * the prefix -- it is an input to the Xcode phase, not part of what the app carries. * diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index 823db419d..0eb32b7bd 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -159,7 +159,11 @@ interface PythonBindingsExtension { /** python-build-standalone release tag to stage from. Defaults to the one this library used. */ val pythonBuildStandaloneRelease: Property - /** Whether to stage a free-threaded build. Defaults to what this library was built against. */ + /** + * Whether to stage a free-threaded build. Defaults to `true`, what the desktop library embeds + * (issue #158: desktop is free-threaded CPython 3.15t only). `false` stages a GIL prefix whose + * `lib/python3.15/` the desktop jar's `libpython3.15t` does not look for. + */ val pythonFreeThreaded: Property /** @@ -241,6 +245,25 @@ interface PythonBindingsExtension { /** The `ios { }` block itself: without it the nested property is reachable only as `ios.x`. */ fun ios(action: org.gradle.api.Action) = action.execute(ios) + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): directory of developer `.py` files to compile + * to C with `python -m typedpython emit-c`. When set, registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]); unset (the default) registers no task. + */ + val typedPythonSourceDir: org.gradle.api.file.DirectoryProperty + + /** + * The directory module names are derived relative to (`--root`). Defaults to + * [typedPythonSourceDir]. + */ + val typedPythonRootDir: org.gradle.api.file.DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + val typedPythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in [typedPythonExecutable]. */ + val typedPythonPath: org.gradle.api.file.ConfigurableFileCollection } /** @@ -326,6 +349,34 @@ class PythonBindingsPlugin : Plugin { // `plugins { ... }` and so do not exist while this plugin is being applied. configureArtifactBindings(this, extension) configureWasmProxyExports(this) + configureTypedPythonEmitC(this, extension) + } + } + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]) when [PythonBindingsExtension.typedPythonSourceDir] is set. + * + * A no-op when [PythonBindingsExtension.typedPythonSourceDir] is unset, so projects that do not + * compile developer Python modules with TypedPython get no task. + */ + internal fun configureTypedPythonEmitC(project: Project, extension: PythonBindingsExtension) { + if (!extension.typedPythonSourceDir.isPresent) return + val srcDir = extension.typedPythonSourceDir.get().asFile + project.tasks.register(EMIT_TYPEDPYTHON_C_TASK, TypedPythonEmitCTask::class.java) { + group = "python" + description = "Runs python -m typedpython emit-c over pythonBindings.typedPythonSourceDir and writes the compile-slot manifest." + sources.from( + project.fileTree(srcDir) { + include("**/*.py") + exclude("**/__pycache__/**", "**/.*", "**/build/**") + }, + ) + rootDirectory.set(extension.typedPythonRootDir.orElse(extension.typedPythonSourceDir)) + pythonExecutable.set(extension.typedPythonExecutable.orElse("python3")) + pythonPath.from(extension.typedPythonPath) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt new file mode 100644 index 000000000..7e5d2e28f --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -0,0 +1,362 @@ +package python.multiplatform.gradle + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.api.tasks.TaskAction +import java.io.File + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest format). */ +const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ +const val TYPEDPYTHON_EMIT_C_VERSION = 1 + +/** Task name registered by [PythonBindingsPlugin] when [PythonBindingsExtension.typedPythonSourceDir] is set. */ +const val EMIT_TYPEDPYTHON_C_TASK = "emitTypedPythonC" + +/** + * One extension module's generated C and header/define metadata for pypackpack's compile slot + * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). + */ +data class ExtensionModuleSource( + val moduleName: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), +) + +/** + * One function or class that stayed interpreted, and why (`pypackpack` `docs/design/compile-slot.md` section 2). + * + * @param file path relative to `projectRoot`, `/`-separated. + * @param line 1-based line number, or `null` when unavailable. + */ +data class SkippedItem( + val name: String, + val file: String, + val line: Int?, + val message: String, +) + +/** + * One developer module for which TypedPython produced no C, and why (`pypackpack` `BundleRequest.notNativeModules`). + */ +data class NotNativeModule( + val module: String, + val kind: Kind, + val items: List = emptyList(), +) { + enum class Kind { + NOT_MARKED, + NOTHING_COMPILED, + } +} + +/** + * One entry of `nativeModules` in the `emit-c` manifest (`docs/design/typedpython.md` section 4.3.2a), + * carrying both the compile-slot fields ([toExtensionModuleSource]) and the reporting fields + * ([file], [status], [skippedItems]). + */ +data class TypedPythonNativeModuleEntry( + val moduleName: String, + val file: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), + val status: String, + val skippedItems: List = emptyList(), +) { + fun toExtensionModuleSource(): ExtensionModuleSource = + ExtensionModuleSource( + moduleName = moduleName, + sources = sources, + includeDirs = includeDirs, + defines = defines, + ) +} + +/** + * Parsed `typedpython-emit-c` v1 manifest (`docs/design/typedpython.md` section 4.3.2a, SPEC N-10). + */ +data class TypedPythonEmitCManifest( + val format: String, + val version: Int, + val compilerVersion: String, + val projectRoot: File, + val runtimeInclude: File, + val nativeModules: List, + val notNativeModules: List, +) { + /** Maps [nativeModules] onto pypackpack's `BundleRequest.nativeModules`. */ + fun toExtensionModuleSources(): List = + nativeModules.map { it.toExtensionModuleSource() } + + /** Maps [notNativeModules] onto pypackpack's `BundleRequest.notNativeModules`. */ + fun toNotNativeModules(): List = notNativeModules +} + +/** + * Parses a `typedpython-emit-c` v1 JSON manifest (`docs/design/typedpython.md` section 4.3.2a) + * into [TypedPythonEmitCManifest]. + * + * Unknown top-level or entry keys are ignored as specified in section 4.3.2a ("adding keys does not + * bump the version"). Unknown `format`, `version` or `notNativeModules[i].kind` fails with + * [IllegalArgumentException]. + */ +fun parseTypedPythonEmitCManifest(json: String): TypedPythonEmitCManifest { + val root = try { + JsonSlurper().parseText(json) + } catch (e: Exception) { + throw IllegalArgumentException("typedpython emit-c manifest is not valid JSON: ${e.message}", e) + } as? Map<*, *> ?: throw IllegalArgumentException("typedpython emit-c manifest is not a JSON object.") + + val format = root["format"] as? String + require(format == TYPEDPYTHON_EMIT_C_FORMAT) { + "typedpython emit-c manifest has format '$format'; expected '$TYPEDPYTHON_EMIT_C_FORMAT'." + } + val version = (root["version"] as? Number)?.toInt() + require(version == TYPEDPYTHON_EMIT_C_VERSION) { + "typedpython emit-c manifest has version '${root["version"]}'; expected version $TYPEDPYTHON_EMIT_C_VERSION." + } + + fun requireString(map: Map<*, *>, key: String, owner: String): String = + (map[key] as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.$key' is missing or empty.") + + fun parseSkippedItems(rawList: Any?, owner: String): List { + val list = rawList as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' must be a list.") + return list.mapIndexed { idx, elem -> + val item = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx]' is not an object.") + val lineVal = item["line"] + val line = when (lineVal) { + null -> null + is Number -> lineVal.toInt() + else -> throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner[$idx].line' must be an integer or null, got '$lineVal'.", + ) + } + SkippedItem( + name = requireString(item, "name", "$owner[$idx]"), + file = requireString(item, "file", "$owner[$idx]"), + line = line, + message = (item["message"] as? String) + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx].message' is missing."), + ) + } + } + + val rawNative = root["nativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'nativeModules' is missing or not a list.") + val nativeModules = rawNative.mapIndexed { idx, elem -> + val owner = "nativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val sourcesRaw = map["sources"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' is missing or not a list.") + val sources = sourcesRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' contains a non-string entry.") + File(str) + } + val includeDirsRaw = map["includeDirs"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' is missing or not a list.") + val includeDirs = includeDirsRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' contains a non-string entry.") + File(str) + } + val definesRaw = map["defines"] as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' is missing or not an object.") + val defines = definesRaw.entries.associate { (k, v) -> + val key = (k as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' has a blank key.") + val value = when (v) { + null -> null + is String -> v + else -> v.toString() + } + key to value + } + TypedPythonNativeModuleEntry( + moduleName = requireString(map, "moduleName", owner), + file = requireString(map, "file", owner), + sources = sources, + includeDirs = includeDirs, + defines = defines, + status = requireString(map, "status", owner), + skippedItems = parseSkippedItems(map["skippedItems"] ?: emptyList(), "$owner.skippedItems"), + ) + } + + val rawNotNative = root["notNativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'notNativeModules' is missing or not a list.") + val notNativeModules = rawNotNative.mapIndexed { idx, elem -> + val owner = "notNativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val kindRaw = requireString(map, "kind", owner) + val kind = try { + NotNativeModule.Kind.valueOf(kindRaw) + } catch (e: IllegalArgumentException) { + throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner.kind' has unknown kind '$kindRaw'; " + + "expected one of ${NotNativeModule.Kind.values().joinToString()}.", + e, + ) + } + NotNativeModule( + module = requireString(map, "module", owner), + kind = kind, + items = parseSkippedItems(map["items"] ?: emptyList(), "$owner.items"), + ) + } + + return TypedPythonEmitCManifest( + format = format, + version = version, + compilerVersion = requireString(root, "compilerVersion", "root"), + projectRoot = File(requireString(root, "projectRoot", "root")), + runtimeInclude = File(requireString(root, "runtimeInclude", "root")), + nativeModules = nativeModules, + notNativeModules = notNativeModules, + ) +} + +/** + * Builds the `python -m typedpython emit-c` command line (`docs/design/typedpython.md` section 4.3.2a). + * Input files are sorted by their path relative to [rootDir] so the manifest module order is deterministic. + */ +fun typedPythonEmitCCommand( + pythonExecutable: String, + files: List, + rootDir: File, + cacheDir: File, + manifestFile: File, +): List { + val rootAbs = rootDir.absoluteFile + val sorted = files.map { it.absoluteFile }.sortedBy { file -> + file.relativeToOrSelf(rootAbs).invariantSeparatorsPath + } + return buildList { + add(pythonExecutable) + add("-m") + add("typedpython") + add("emit-c") + sorted.forEach { add(it.absolutePath) } + add("--root") + add(rootAbs.absolutePath) + add("--cache") + add(cacheDir.absoluteFile.absolutePath) + add("--manifest") + add(manifestFile.absoluteFile.absolutePath) + } +} + +/** + * SPEC N-10 (`python-multiplatform#25` step 2): runs `python -m typedpython emit-c` over [sources] + * and writes the per-module C cache ([cacheDirectory]) and the v1 JSON manifest ([manifestFile]) + * for pypackpack's native compile slot. + */ +abstract class TypedPythonEmitCTask : DefaultTask() { + + /** The developer `.py` files to compile to C. */ + @get:InputFiles + @get:SkipWhenEmpty + @get:IgnoreEmptyDirectories + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + /** + * The directory module names are derived relative to (`--root`, the ppp package's `python/` + * or `src/main` root). Fingerprinted through [sources]'s `PathSensitivity.RELATIVE`. + */ + @get:Internal + abstract val rootDirectory: DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + @get:Input + abstract val pythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in the interpreter. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val pythonPath: ConfigurableFileCollection + + /** `generate_project`'s incremental source-to-C cache directory (`--cache`). */ + @get:OutputDirectory + abstract val cacheDirectory: DirectoryProperty + + /** The JSON manifest written by `emit-c` (`--manifest`). */ + @get:OutputFile + abstract val manifestFile: RegularFileProperty + + init { + pythonExecutable.convention("python3") + } + + /** Parses the manifest written at [manifestFile] into [TypedPythonEmitCManifest]. */ + fun readManifest(): TypedPythonEmitCManifest = + parseTypedPythonEmitCManifest(manifestFile.get().asFile.readText()) + + @TaskAction + fun emitC() { + val pyFiles = sources.files.filter { it.isFile && it.extension == "py" } + val root = rootDirectory.get().asFile + val cache = cacheDirectory.get().asFile + val manifest = manifestFile.get().asFile + val executable = pythonExecutable.getOrElse("python3") + + val command = typedPythonEmitCCommand( + pythonExecutable = executable, + files = pyFiles, + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + logger.info("typedPythonEmitC: ${command.joinToString(" ")}") + + val work = temporaryDir.apply { mkdirs() } + val tmp = File(work, "tmp").apply { mkdirs() } + val stdoutFile = File(work, "emit-c.stdout") + val stderrFile = File(work, "emit-c.stderr") + + val pb = ProcessBuilder(command) + .directory(project.projectDir) + .redirectOutput(stdoutFile) + .redirectError(stderrFile) + pb.environment()["TMPDIR"] = tmp.absolutePath + val extraPythonPath = pythonPath.files.map { it.absolutePath }.filter { it.isNotEmpty() } + if (extraPythonPath.isNotEmpty()) { + val existing = pb.environment()["PYTHONPATH"].orEmpty() + val joined = extraPythonPath.joinToString(File.pathSeparator) + pb.environment()["PYTHONPATH"] = if (existing.isEmpty()) joined else "$joined${File.pathSeparator}$existing" + } + + val exitCode = pb.start().waitFor() + if (exitCode != 0) { + val details = listOf(stderrFile.readText().trim(), stdoutFile.readText().trim()) + .filter { it.isNotEmpty() } + .joinToString("\n") + .ifEmpty { "(no output)" } + throw GradleException("typedpython emit-c failed (exit $exitCode):\n$details") + } + } +} diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt index 70babf976..dabd6980d 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt @@ -462,7 +462,8 @@ internal object ArtifactScanner { functions = functionsOf(metadata.kmPackage), ownerNode = node, classpath = classpath, - ) + extensionPropertyCandidates(metadata.kmPackage, node, owner, classpath) + ) + extensionPropertyCandidates(metadata.kmPackage, node, owner, classpath) + + constantsOf(metadata.kmPackage.properties, node, boundOwner = owner, callOwner = owner, classpath = classpath, ownerIsClass = false) } is KotlinClassMetadata.MultiFileClassFacade -> metadata.partClassNames.flatMap { partBinaryName -> val partNode = classpath.classNode(partBinaryName) ?: return@flatMap emptyList() @@ -475,7 +476,8 @@ internal object ArtifactScanner { functions = functionsOf(partMetadata.kmPackage), ownerNode = partNode, classpath = classpath, - ) + extensionPropertyCandidates(partMetadata.kmPackage, partNode, owner, classpath) + ) + extensionPropertyCandidates(partMetadata.kmPackage, partNode, owner, classpath) + + constantsOf(partMetadata.kmPackage.properties, partNode, boundOwner = owner, callOwner = owner, classpath = classpath, ownerIsClass = false) } // A lone part is package-private and never reaches here on its own (the `ACC_PUBLIC` // guard above already excluded it) -- only through the facade case, which fetches it by @@ -875,7 +877,23 @@ internal object ArtifactScanner { boundOwner: String, callOwner: String, classpath: ArtifactClasspath, - ): List = kmClass.properties.mapNotNull { property -> + ): List = constantsOf( + properties = kmClass.properties, + declaringNode = declaringNode, + boundOwner = boundOwner, + callOwner = callOwner, + classpath = classpath, + ownerIsClass = true, + ) + + private fun constantsOf( + properties: List, + declaringNode: ClassNode, + boundOwner: String, + callOwner: String, + classpath: ArtifactClasspath, + ownerIsClass: Boolean = true, + ): List = properties.mapNotNull { property -> if (property.visibility != Visibility.PUBLIC) return@mapNotNull null if (property.receiverParameterType != null) return@mapNotNull null // A `@Composable get()` reads the composition, so its value does not exist outside one -- @@ -928,7 +946,7 @@ internal object ArtifactScanner { declaration = DeclarationModel( simpleName = property.name, owner = boundOwner, - ownerIsClass = true, + ownerIsClass = ownerIsClass, receiver = null, parameters = emptyList(), returnType = returnModel, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt index 0460c5db3..96ef89ddf 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt @@ -280,22 +280,49 @@ private class StubRenderer(declarations: List) { /** Class qualified name -> its bound constructors, in table-key order; filled by [renderModuleDefs]. */ private val constructorsByClass = sortedMapOf>() + private val supertypes: Map> = bound + .filter { it.returnSupertypes.isNotEmpty() } + .groupBy { it.returnType.qualifiedName } + .mapValues { (_, entries) -> entries.first().returnSupertypes } + /** - * Module prefixes (every module and every directory above it) that name one directory on a - * case-insensitive filesystem together with a sibling (issue #44). In each such group the - * all-lower-case spelling keeps its path -- a Kotlin package -- and every other spelling is a - * class module that is emitted inside its parent package's `__init__.pyi` instead. + * Module prefixes folded into their parent package's `__init__.pyi`. + * + * Two reasons fold a class module into its parent: + * (a) Case-insensitive path collision with a sibling (issue #44): in each such group the + * all-lower-case spelling keeps its path -- a Kotlin package -- and every other spelling + * is a class module emitted inside its parent package's `__init__.pyi`. + * (b) A receiver class (with instance methods or properties and a companion handle) whose companion + * module only publishes constants (`Material3DesignSystem.Dark`), where the runtime + * (`class_wins_over_module`, issue #322) serves the class on the parent package and the constants + * on the class, provided `module` is not a supertype of another bound type (so an interface like + * `DrawScope` stays a module while a concrete receiver class like `Material3DesignSystem` is folded). */ private val foldedPrefixes: Set = run { val prefixes = byModule.keys.flatMap { module -> val parts = module.split('.') parts.indices.map { parts.take(it + 1).joinToString(".") } }.toSet() - prefixes.groupBy { it.lowercase() }.values + val caseColliding = prefixes.groupBy { it.lowercase() }.values .filter { it.size > 1 } .flatten() .filter { '.' in it && it.substringAfterLast('.') != it.substringAfterLast('.').lowercase() } .toSet() + val allSupertypes = supertypes.values.flatten().toSet() + val companionConstantClasses = byModule.keys.filter { module -> + module !in allSupertypes && + module in companionHandles && + (module in methods || module in properties) && + byModule[module]?.all { it.kind == "STATIC_GETTER" } == true && + module.substringAfterLast('.') !in takenNames[module.substringBeforeLast('.', "")].orEmpty() && + run { + val prefix = "$module." + bound.none { it.returnType.qualifiedName.startsWith(prefix) || it.owner.startsWith(prefix) } && + methods.keys.none { it.startsWith(prefix) } && + properties.keys.none { it.startsWith(prefix) } + } + }.toSet() + caseColliding + companionConstantClasses } /** The shortest folded prefix of [module] (the module itself included), or `null` if it keeps its own path. */ @@ -307,11 +334,6 @@ private class StubRenderer(declarations: List) { /** The modules that are still files: a folded class module is a class, not a module, for `classReference`. */ private val moduleNames: Set = byModule.keys.filter { foldedPrefixOf(it) == null }.toSet() - private val supertypes: Map> = bound - .filter { it.returnSupertypes.isNotEmpty() } - .groupBy { it.returnType.qualifiedName } - .mapValues { (_, entries) -> entries.first().returnSupertypes } - // ----------------------------------------------------------------------- per-module output private class ClassNode(val name: String) { diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt index c0dc4a075..7daae5b07 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt @@ -36,7 +36,7 @@ class AcquiredHeaderVersionTest { try { for (t in CPythonIncludeLayout.allTargets) { val v = if (t.startsWith("ios")) "3.14.6" else "3.14.7" - root.resolve(CPythonIncludeLayout.relativePath(t, CPythonFlavour.GIL, "3.14.7")) + root.resolve(CPythonIncludeLayout.relativePath(t, CPythonIncludeLayout.embeddedFlavour(t), "3.14.7")) .also { it.mkdirs() }.resolve("patchlevel.h").writeText(patchlevel(v)) } val failures = AcquiredHeaderVersion.mismatches(root, "3.14.7") diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt index b7b199545..d803ee893 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt @@ -22,13 +22,14 @@ class CPythonIncludeDirectoriesTest { assertEquals("linux-x86_64-freethreaded/python/include/python3.14t", l.relativePath("linux-x86_64", CPythonFlavour.FREE_THREADED, version)) assertEquals("windows-x86_64/python/include", l.relativePath("windows-x86_64", CPythonFlavour.GIL, version)) assertEquals("windows-x86_64-freethreaded/python/include", l.relativePath("windows-x86_64", CPythonFlavour.FREE_THREADED, version)) - // #313: the 3.15 iOS archive has headers only inside the framework. - assertEquals("ios/Python.xcframework/ios-arm64/Python.framework/Headers", l.relativePath("ios-arm64", CPythonFlavour.GIL, version)) + // #313: the iOS archive has headers only inside the framework. + assertEquals("ios/Python.xcframework/ios-arm64/Python.framework/Headers", l.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version)) assertEquals( "ios/Python.xcframework/ios-arm64_x86_64-simulator/Python.framework/Headers", - l.relativePath("ios-arm64_x86_64-simulator", CPythonFlavour.GIL, version), + l.relativePath("ios-arm64_x86_64-simulator", CPythonFlavour.FREE_THREADED, version), ) - assertEquals("android-aarch64/prefix/include/python3.14", l.relativePath("android-aarch64", CPythonFlavour.GIL, version)) + // #328: the 3.15t Android archive (`prefix/include/python3.15t`), unpacked beside a `-freethreaded` suffix. + assertEquals("android-aarch64-freethreaded/prefix/include/python3.14t", l.relativePath("android-aarch64", CPythonFlavour.FREE_THREADED, version)) } @Test @@ -41,12 +42,22 @@ class CPythonIncludeDirectoriesTest { } } + /** + * Issue #328: Android and iOS embed only the project's own 3.15t builds; there is no GIL mobile + * build to lay out. Red before: `supports` allowed only GIL off desktop, so the free-threaded + * assertions below failed, and the GIL ones passed the other way round. + */ @Test - fun freeThreadedDoesNotExistOffDesktop() { - assertFalse(CPythonIncludeLayout.supports("android-aarch64", CPythonFlavour.FREE_THREADED)) - assertFalse(CPythonIncludeLayout.supports("ios-arm64", CPythonFlavour.FREE_THREADED)) + fun mobileHasOnlyTheFreeThreadedBuild() { + for (t in CPythonIncludeLayout.androidTargets + CPythonIncludeLayout.iosTargets) { + assertTrue(CPythonIncludeLayout.supports(t, CPythonFlavour.FREE_THREADED), t) + assertFalse(CPythonIncludeLayout.supports(t, CPythonFlavour.GIL), t) + } + assertFailsWith { + CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.GIL, version) + } assertFailsWith { - CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version) + CPythonIncludeLayout.relativePath("android-x86_64", CPythonFlavour.GIL, version) } } @@ -62,8 +73,8 @@ class CPythonIncludeDirectoriesTest { val project = ProjectBuilder.builder().build() val root = project.layout.projectDirectory.dir("extracted/$version") val tasks = CPythonIncludeLayout.allTargets.associateWith { target -> - // Android and iOS are always GIL; desktop follows the configured flavour. - val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + // Android and iOS are always free-threaded (#328); desktop follows the configured flavour. + val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.FREE_THREADED project.tasks.register("fake_${target.replace('-', '_')}") { doLast { root.dir(CPythonIncludeLayout.relativePath(target, flavour, version)).asFile @@ -72,7 +83,7 @@ class CPythonIncludeDirectoriesTest { } } val dirs = CPythonIncludeDirectories(root, version) { target, flavour -> - val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.FREE_THREADED if (flavour == extracted) tasks[target] else null } return project to dirs @@ -80,7 +91,8 @@ class CPythonIncludeDirectoriesTest { @Test fun everyTargetResolvesToADistinctDirectoryHoldingPythonH() { - val (project, dirs) = fixture(CPythonFlavour.GIL) + // FREE_THREADED is the desktop flavour the library build extracts (#158). + val (project, dirs) = fixture(CPythonFlavour.FREE_THREADED) val seen = mutableSetOf() for (t in dirs.targets) { val producer = project.tasks.getByName("fake_${t.replace('-', '_')}") @@ -99,12 +111,32 @@ class CPythonIncludeDirectoriesTest { } @Test - fun theOtherFlavourFailsLoudlyNamingTheProperty() { - val (_, dirs) = fixture(CPythonFlavour.GIL) - assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.FREE_THREADED)) - val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.FREE_THREADED) } - assertTrue("pythonFreeThreaded" in e.message.orEmpty()) - assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.FREE_THREADED) } + fun theOtherFlavourFailsLoudlyNamingTheDecision() { + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) + assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.GIL)) + val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.GIL) } + assertTrue("#158" in e.message.orEmpty(), e.message) + val ios = assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.GIL) } + assertTrue("#328" in ios.message.orEmpty(), ios.message) + } + + /** + * Issues #158 and #328: every target embeds only the free-threaded build, so the default + * flavour of every accessor is free-threaded. Red before #328: `embeddedFlavour` was GIL for + * Android and iOS, so the second loop and the Android path below failed. + */ + @Test + fun theDefaultFlavourIsTheOneTheLibraryEmbeds() { + for (t in CPythonIncludeLayout.allTargets) { + assertEquals(CPythonFlavour.FREE_THREADED, CPythonIncludeLayout.embeddedFlavour(t), t) + } + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) + assertTrue(dirs.isAvailable("macos-aarch64")) + val root = dirs.includeDir("macos-aarch64").get().asFile.parentFile.parentFile.parentFile.parentFile + assertEquals(File(root, "macos-aarch64-freethreaded/python/include/python3.14t"), dirs.includeDir("macos-aarch64").get().asFile) + assertEquals(File(root, "android-aarch64-freethreaded/prefix/include/python3.14t"), dirs.includeDir("android-aarch64").get().asFile) + assertEquals("python314t.lib", dirs.libraryFile("windows-x86_64").get().asFile.name) + assertEquals("libpython3.14t.so", dirs.libraryFile("android-x86_64").get().asFile.name) } @Test @@ -119,11 +151,11 @@ class CPythonIncludeDirectoriesTest { @Test fun linkLibraryLayoutMatchesTheObservedTree() { val l = CPythonIncludeLayout - assertEquals("android-aarch64/prefix/lib", l.linkLibraryDirRelativePath("android-aarch64", CPythonFlavour.GIL)) - assertEquals("android-x86_64/prefix/lib", l.linkLibraryDirRelativePath("android-x86_64", CPythonFlavour.GIL)) + assertEquals("android-aarch64-freethreaded/prefix/lib", l.linkLibraryDirRelativePath("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertEquals("android-x86_64-freethreaded/prefix/lib", l.linkLibraryDirRelativePath("android-x86_64", CPythonFlavour.FREE_THREADED)) assertEquals("windows-x86_64/python/libs", l.linkLibraryDirRelativePath("windows-x86_64", CPythonFlavour.GIL)) assertEquals("windows-x86_64-freethreaded/python/libs", l.linkLibraryDirRelativePath("windows-x86_64", CPythonFlavour.FREE_THREADED)) - assertEquals("libpython3.14.so", l.linkLibraryFileName("android-aarch64", CPythonFlavour.GIL, version)) + assertEquals("libpython3.14t.so", l.linkLibraryFileName("android-aarch64", CPythonFlavour.FREE_THREADED, version)) assertEquals("python314.lib", l.linkLibraryFileName("windows-x86_64", CPythonFlavour.GIL, version)) assertEquals("python314t.lib", l.linkLibraryFileName("windows-x86_64", CPythonFlavour.FREE_THREADED, version)) } @@ -138,17 +170,20 @@ class CPythonIncludeDirectoriesTest { } @Test - fun linkLibraryFreeThreadedOnlyOnWindows() { + fun linkLibraryAndroidIsFreeThreadedOnly() { val l = CPythonIncludeLayout assertTrue(l.supportsLinkLibrary("windows-x86_64", CPythonFlavour.FREE_THREADED)) - assertFalse(l.supportsLinkLibrary("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertTrue(l.supportsLinkLibrary("windows-x86_64", CPythonFlavour.GIL)) + // #328: Android links `libpython3.15t.so`; there is no GIL Android build. + assertTrue(l.supportsLinkLibrary("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertFalse(l.supportsLinkLibrary("android-x86_64", CPythonFlavour.GIL)) assertFalse(l.supportsLinkLibrary("macos-aarch64", CPythonFlavour.GIL)) assertFailsWith { l.linkLibraryDirRelativePath("macos-aarch64", CPythonFlavour.GIL) } } @Test fun linkLibraryProvidersCarryTheExtractionTaskAndResolve() { - val (project, dirs) = fixture(CPythonFlavour.GIL) + val (project, dirs) = fixture(CPythonFlavour.FREE_THREADED) val root = project.layout.projectDirectory.dir("extracted/$version") for (t in dirs.targets.filter { dirs.isLinkRequired(it) }) { val producer = project.tasks.getByName("fake_${t.replace('-', '_')}") @@ -159,24 +194,25 @@ class CPythonIncludeDirectoriesTest { val fileConsumer = project.tasks.register("linkf_${t.replace('-', '_')}") { inputs.file(fileProvider) }.get() assertTrue(producer in fileConsumer.taskDependencies.getDependencies(fileConsumer), "$t: libraryFile lacks task") // Simulate the extraction creating the library. - val expected = root.dir(CPythonIncludeLayout.linkLibraryDirRelativePath(t, CPythonFlavour.GIL)).asFile + val flavour = CPythonIncludeLayout.embeddedFlavour(t) + val expected = root.dir(CPythonIncludeLayout.linkLibraryDirRelativePath(t, flavour)).asFile expected.mkdirs() - File(expected, CPythonIncludeLayout.linkLibraryFileName(t, CPythonFlavour.GIL, version)).writeText("lib") + File(expected, CPythonIncludeLayout.linkLibraryFileName(t, flavour, version)).writeText("lib") assertEquals(expected, dirProvider.get().asFile) - assertEquals(expected.resolve(CPythonIncludeLayout.linkLibraryFileName(t, CPythonFlavour.GIL, version)), fileProvider.get().asFile) + assertEquals(expected.resolve(CPythonIncludeLayout.linkLibraryFileName(t, flavour, version)), fileProvider.get().asFile) } } @Test fun linkLibraryFailsLoudlyWhenNotRequiredOrWrongFlavour() { - val (_, dirs) = fixture(CPythonFlavour.GIL) + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) assertFalse(dirs.isLinkRequired("macos-aarch64")) assertFailsWith { dirs.libraryDir("macos-aarch64") } assertFailsWith { dirs.libraryFile("ios-arm64") } - assertFalse(dirs.isLinkAvailable("windows-x86_64", CPythonFlavour.FREE_THREADED)) - val e = assertFailsWith { dirs.libraryDir("windows-x86_64", CPythonFlavour.FREE_THREADED) } - assertTrue("pythonFreeThreaded" in e.message.orEmpty()) - assertFailsWith { dirs.libraryDir("android-aarch64", CPythonFlavour.FREE_THREADED) } + assertFalse(dirs.isLinkAvailable("windows-x86_64", CPythonFlavour.GIL)) + val e = assertFailsWith { dirs.libraryDir("windows-x86_64", CPythonFlavour.GIL) } + assertTrue("#158" in e.message.orEmpty(), e.message) + assertFailsWith { dirs.libraryDir("android-aarch64", CPythonFlavour.GIL) } } @Test diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt index 09d1c5512..442b6d1d2 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt @@ -52,6 +52,34 @@ class EmbeddedPythonVersionTest { } } + /** + * Issue #158: desktop embeds 3.15t while Android and iOS embed the GIL 3.15 until #328, so the + * free-threaded attribute is decided per variant. Red before: `register` took no per-variant + * function and stamped one value on every variant, so the AAR's variants claimed what the + * desktop jar's did (this test then does not compile, which is the pre-implementation state). + */ + @Test + fun freeThreadedIsDecidedPerVariant() { + val project = ProjectBuilder.builder().build() + val names = listOf("desktopRuntimeElements", "releaseRuntimeElements", "iosArm64ApiElements", "metadataApiElements") + val configs = names.associateWith { n -> project.configurations.create(n) { isCanBeConsumed = true } } + EmbeddedPythonVersion("3.15.0", true).register(project) { name -> + when { + name.startsWith("metadata") -> null + name.startsWith("desktop") -> true + else -> false + } + } + val ft = attr(EmbeddedPythonVersion.ATTRIBUTE_FREE_THREADED) + assertEquals("true", configs.getValue("desktopRuntimeElements").attributes.getAttribute(ft)) + assertEquals("false", configs.getValue("releaseRuntimeElements").attributes.getAttribute(ft)) + assertEquals("false", configs.getValue("iosArm64ApiElements").attributes.getAttribute(ft)) + assertNull(configs.getValue("metadataApiElements").attributes.getAttribute(ft)) + for (c in configs.values) { + assertEquals("3.15.0", c.attributes.getAttribute(attr(EmbeddedPythonVersion.ATTRIBUTE_VERSION))) + } + } + @Test fun nonConsumableConfigurationsAreLeftAlone() { val project = ProjectBuilder.builder().build() diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt index 645ee524e..3d7de7808 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt @@ -11,7 +11,6 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse -import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue @@ -147,8 +146,8 @@ class IosConsumerWiringTest { val build = project.layout.buildDirectory.get().asFile assertEquals(File(build, "python-standalone"), task.workDirectory.get().asFile) assertTrue(task.extractDirectory.startsWith(File(build, "python-standalone"))) - assertEquals(DEFAULT_IOS_ARCHIVE_SHA256, task.expectedSha256.get()) - assertEquals(64, DEFAULT_IOS_ARCHIVE_SHA256.length) + assertEquals(DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256, task.expectedSha256.get()) + assertEquals(64, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256.length) } @Test @@ -160,6 +159,49 @@ class IosConsumerWiringTest { assertEquals(File(build, "python-ios-home/${IosPythonHomeLayout.slices[0].name}"), stage.destinationDir.get().asFile) } + // ---- the 3.15t archive is the only one (SPEC T-2a, owner decision #158) -------------------- + + @Test + fun theDefaultIsTheProjectsOwn315tArchiveAndLayout() { + val project = project() + wire(project) + val v = DEFAULT_PYTHON_VERSION + val task = project.tasks.getByName(DOWNLOAD_IOS_TASK) as DownloadPythonIosTask + assertEquals( + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-${v}t-mobile/python-${v}t-iOS-XCframework.tar.gz", + task.archiveUrl.get(), + ) + assertEquals("python-${v}t-iOS-XCframework.tar.gz", task.archiveName.get()) + assertEquals(DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256, task.expectedSha256.get()) + for (slice in IosPythonHomeLayout.slices) { + val stage = project.tasks.getByName(slice.stageTaskName) as StageIosPythonHomeTask + assertEquals(v, stage.pythonVersion.get(), slice.name) + } + assertTrue(IosPythonHomeLayout.stdlibTag(v).endsWith("t"), "the staged stdlib is lib/pythont/") + } + + @Test + fun theFreeThreadedPinIsTheReleasesDigest() { + // The value the release's SHA256SUMS and the GitHub asset digest both give for + // python-3.15.0t-iOS-XCframework.tar.gz (cpython-3.15.0t-mobile, run 39014667720, #163). + assertEquals("ios-$DEFAULT_PYTHON_VERSION-freethreaded", DEFAULT_IOS_FREE_THREADED_LOCK_KEY) + assertEquals(DEFAULT_IOS_FREE_THREADED_LOCK_KEY, FreeThreadedMobileRelease.lockKey("ios", DEFAULT_PYTHON_VERSION)) + if (DEFAULT_PYTHON_VERSION == "3.15.0") { + assertEquals("b8c0a4b32d31738f6ead7ace28acee0cb2a590bd0ca7015e9c32af613445e6b5", DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) + } + } + + @Test + fun theReleaseNamesFollowTheTSuffix() { + assertEquals("cpython-3.15.0t-mobile", FreeThreadedMobileRelease.tag("3.15.0")) + assertEquals("python-3.15.0t-iOS-XCframework.tar.gz", FreeThreadedMobileRelease.iosArchiveName("3.15.0")) + assertEquals( + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-3.15.0t-mobile/python-3.15.0t-aarch64-linux-android.tar.gz", + FreeThreadedMobileRelease.url("3.15.0", FreeThreadedMobileRelease.androidArchiveName("3.15.0", "aarch64")), + ) + assertEquals("android-aarch64-3.15.0-freethreaded", FreeThreadedMobileRelease.lockKey("android-aarch64", "3.15.0")) + } + // ---- download task ------------------------------------------------------------------------ private fun tarGz(entries: Map): ByteArray { @@ -221,6 +263,16 @@ class IosConsumerWiringTest { assertFalse(File(task.extractDirectory, "Python.xcframework").exists()) } + @Test + fun aMissingPinFailsBeforeAnyDownload() { + // A plugin built from a python-checksums.properties without the iOS key bakes "". + val bytes = tarGz(mapOf("Python.xcframework/Info.plist" to "plist")) + val (task, _) = downloadTask(bytes, "") + val e = assertFailsWith { task.provision() } + assertTrue("no pinned SHA-256" in e.message.orEmpty(), e.message) + assertFalse(File(task.workDirectory.get().asFile, "fake-ios.tar.gz").exists()) + } + // ---- script ------------------------------------------------------------------------------- @Test @@ -249,6 +301,5 @@ class IosConsumerWiringTest { assertEquals("ios-arm64_x86_64-simulator", xcframeworkSliceForKonanTarget("ios_simulator_arm64")) assertEquals("ios-arm64_x86_64-simulator", xcframeworkSliceForKonanTarget("ios_x64")) assertNull(xcframeworkSliceForKonanTarget("linux_x64")) - assertNotNull(DEFAULT_IOS_ARCHIVE_URL) } } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt index 7f11258e3..a96b4a111 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt @@ -19,12 +19,12 @@ import kotlin.test.assertTrue * The two share no code, so each pins the directory name as a literal. * * What this cannot find: whether CPython on a device really loads the result. The fake tree below - * mirrors the observed 3.14 archive's *shape*; the installed-app run in + * mirrors the observed 3.15.0t archive's *shape*; the installed-app run in * `docs/platforms/ios-app-bundle.md` is the check for the rest. */ class IosPythonHomeLayoutTest { - private val version = "3.14.7" + private val version = "3.15.0" // Under this module's build directory (the test task's working directory), not the system // temporary directory: AGENTS.md rule 2. private val scratch: File = @@ -94,7 +94,7 @@ class IosPythonHomeLayoutTest { fun theSharedStdlibComesFirstAndTheSliceTreeSecond() { val sim = IosPythonHomeLayout.sliceFor("-iphonesimulator", "x86_64") assertEquals( - listOf("lib/python3.14", "ios-arm64_x86_64-simulator/lib-x86_64/python3.14"), + listOf("lib/python3.15t", "ios-arm64_x86_64-simulator/lib-x86_64/python3.15t"), IosPythonHomeLayout.sourceDirectories(sim, version), ) } @@ -105,8 +105,8 @@ class IosPythonHomeLayoutTest { assertTrue(staged("os.py")) assertTrue(staged("encodings/__init__.py")) assertTrue(staged("json/decoder.py")) - assertTrue(staged("lib-dynload/_json.cpython-314-iphonesimulator.so")) - assertTrue(staged("_sysconfigdata__ios_arm64-iphonesimulator.py")) + assertTrue(staged("lib-dynload/_json.cpython-315t-iphonesimulator.so")) + assertTrue(staged("_sysconfigdata_t_ios_arm64-iphonesimulator.py")) assertTrue(staged("site-packages/README.txt")) // A package *named like* an excluded one deeper down is not excluded. assertTrue(staged("email/test/__init__.py")) @@ -117,9 +117,9 @@ class IosPythonHomeLayoutTest { assertFalse(staged("idlelib/idle.py")) assertFalse(staged("turtledemo/__main__.py")) assertFalse(staged("ensurepip/_bundled/pip.whl")) - assertFalse(staged("encodings/__pycache__/aliases.cpython-314.pyc")) + assertFalse(staged("encodings/__pycache__/aliases.cpython-315t.pyc")) assertFalse(staged("__pycache__")) - assertFalse(staged("libpython3.14.dylib")) + assertFalse(staged("libpython3.15t.dylib")) } @Test @@ -129,10 +129,10 @@ class IosPythonHomeLayoutTest { val plan = IosPythonHomeLayout.plan(xcframework, slice, version) assertEquals( listOf( - "lib/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py", - "lib/python3.14/encodings/__init__.py", - "lib/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so", - "lib/python3.14/os.py", + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", + "lib/python3.15t/os.py", ), plan.keys.toList(), ) @@ -160,7 +160,7 @@ class IosPythonHomeLayoutTest { .build() val out = File(scratch, "out") // Left over from a previous run with a wider selection; must not survive. - File(out, "home/lib/python3.14/test/stale.py").apply { parentFile.mkdirs(); writeText("") } + File(out, "home/lib/python3.15t/test/stale.py").apply { parentFile.mkdirs(); writeText("") } val task = project.tasks.create("stage", StageIosPythonHomeTask::class.java) task.xcframework.set(xcframework) @@ -173,10 +173,10 @@ class IosPythonHomeLayoutTest { .map { it.relativeTo(File(out, "home")).invariantSeparatorsPath }.sorted().toList() assertEquals( listOf( - "lib/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py", - "lib/python3.14/encodings/__init__.py", - "lib/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so", - "lib/python3.14/os.py", + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", + "lib/python3.15t/os.py", ), staged, ) @@ -184,22 +184,42 @@ class IosPythonHomeLayoutTest { assertFalse(File(out, "home/${StageIosPythonHomeTask.TEMPLATE_NAME}").exists()) } - /** The observed 3.14 archive's shape, a few files deep. */ + // ---- the layout is always free-threaded (SPEC T-2a, owner decision #158) ---- + + @Test + fun theStdlibTagAlwaysCarriesTheTSuffix() { + assertEquals("3.15t", IosPythonHomeLayout.stdlibTag("3.15.0")) + assertEquals("3.15t", IosPythonHomeLayout.stdlibTag(version)) + } + + @Test + fun aGilShapedTreeIsNotFoundByThePlan() { + // Guard: a framework with only lib/python3.15 (no t suffix) must fail loudly, not stage. + val root = File(scratch, "gil/Python.xcframework") + fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } + file("lib/python3.15/os.py") + file("ios-arm64/lib-arm64/python3.15/lib-dynload/_json.cpython-315-iphoneos.so") + assertFailsWith { + IosPythonHomeLayout.plan(root, IosPythonHomeLayout.sliceFor("-iphoneos", "arm64"), version) + } + } + + /** The observed 3.15.0t archive's shape, a few files deep. */ private fun fakeXcframework(): File { val root = File(scratch, "Python.xcframework") fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } - file("lib/python3.14/os.py") - file("lib/python3.14/encodings/__init__.py") - file("lib/python3.14/encodings/__pycache__/__init__.cpython-314.pyc") - file("lib/python3.14/test/test_os.py") - file("lib/python3.14/tkinter/__init__.py") - file("ios-arm64/lib-arm64/python3.14/lib-dynload/_json.cpython-314-iphoneos.so") - file("ios-arm64/lib-arm64/python3.14/_sysconfigdata__ios_arm64-iphoneos.py") - file("ios-arm64/lib/libpython3.14.dylib") - file("ios-arm64_x86_64-simulator/lib-arm64/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so") - file("ios-arm64_x86_64-simulator/lib-arm64/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py") - file("ios-arm64_x86_64-simulator/lib-x86_64/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so") - file("ios-arm64_x86_64-simulator/lib/libpython3.14.dylib") + file("lib/python3.15t/os.py") + file("lib/python3.15t/encodings/__init__.py") + file("lib/python3.15t/encodings/__pycache__/__init__.cpython-315t.pyc") + file("lib/python3.15t/test/test_os.py") + file("lib/python3.15t/tkinter/__init__.py") + file("ios-arm64/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphoneos.so") + file("ios-arm64/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py") + file("ios-arm64/lib/libpython3.15t.dylib") + file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") + file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py") + file("ios-arm64_x86_64-simulator/lib-x86_64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") + file("ios-arm64_x86_64-simulator/lib/libpython3.15t.dylib") file("build/iOS-dylib-Info-template.plist") return root } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt index 839ba3751..9e7ab6b92 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt @@ -75,9 +75,24 @@ class PythonHomeStagingTest { ) } + /** + * Issue #158: the desktop library embeds only the free-threaded CPython 3.15t, so the prefix a + * consumer stages by default must be the free-threaded one. Red before: the default was read + * from the root's `pythonFreeThreaded=false`, and the staged `lib/python3.15/` was a directory + * the desktop jar's `libpython3.15t` does not look in. + */ + @Test + fun theStagedPrefixDefaultsToTheFreeThreadedBuild() { + assertTrue(DEFAULT_PYTHON_FREE_THREADED, "desktop is free-threaded only (#158)") + assertEquals( + "cpython-$DEFAULT_PYTHON_VERSION+$DEFAULT_PBS_RELEASE-aarch64-apple-darwin-freethreaded-install_only_stripped.tar.gz", + pbsAssetName(DEFAULT_PYTHON_VERSION, DEFAULT_PBS_RELEASE, "macos-aarch64", DEFAULT_PYTHON_FREE_THREADED), + ) + } + @Test fun aFreeThreadedBuildIsADifferentAssetNotAFlagOnTheSameOne() { - // `-PpythonFreeThreaded=true` selects a different tarball upstream, and the resulting + // The free-threaded build is a different tarball upstream, and the resulting // prefix holds `lib/python3.14t/` rather than `lib/python3.14/`. Staging the default // tarball for a free-threaded consumer produces a prefix `PythonHomeCheck` rejects. assertEquals( diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt new file mode 100644 index 000000000..2ff23ddba --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -0,0 +1,407 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.testfixtures.ProjectBuilder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * SPEC N-10 (python-multiplatform#25 step 2): Gradle task running `python -m typedpython emit-c` + * with declared inputs/outputs and mapping the v1 manifest (`docs/design/typedpython.md` section 4.3.2a) + * onto pypackpack's `BundleRequest.nativeModules` (`ExtensionModuleSource`) and + * `BundleRequest.notNativeModules` (`NotNativeModule` / `SkippedItem`). + */ +class TypedPythonEmitCTaskTest { + + private val sampleManifestJson = + """ + { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "abc123def456", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/site-packages/typedpython/runtime", + "nativeModules": [ + { + "moduleName": "app.physics.nbody", + "file": "app/physics/nbody.py", + "sources": ["/abs/cache/nbody.c"], + "includeDirs": ["/abs/site-packages/typedpython/runtime"], + "defines": {"TP_FLAG": "1", "TP_BARE": null}, + "status": "rebuilt", + "skippedItems": [ + {"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "not pure"} + ] + } + ], + "notNativeModules": [ + { + "module": "app.util", + "file": "app/util.py", + "kind": "NOT_MARKED", + "items": [] + }, + { + "module": "app.text", + "file": "app/text.py", + "kind": "NOTHING_COMPILED", + "items": [ + {"name": "f", "file": "app/text.py", "line": 2, "message": "str slice is not supported"}, + {"name": "g", "file": "app/text.py", "line": null, "message": "no line"} + ] + } + ] + } + """.trimIndent() + + @Test + fun parsesManifestV1AndMapsToBundleRequestShapes() { + val manifest = parseTypedPythonEmitCManifest(sampleManifestJson) + assertEquals(TYPEDPYTHON_EMIT_C_FORMAT, manifest.format) + assertEquals(TYPEDPYTHON_EMIT_C_VERSION, manifest.version) + assertEquals("abc123def456", manifest.compilerVersion) + assertEquals(File("/abs/proj"), manifest.projectRoot) + assertEquals(File("/abs/site-packages/typedpython/runtime"), manifest.runtimeInclude) + + assertEquals(1, manifest.nativeModules.size) + val nativeEntry = manifest.nativeModules.single() + assertEquals("app.physics.nbody", nativeEntry.moduleName) + assertEquals("app/physics/nbody.py", nativeEntry.file) + assertEquals(listOf(File("/abs/cache/nbody.c")), nativeEntry.sources) + assertEquals(listOf(File("/abs/site-packages/typedpython/runtime")), nativeEntry.includeDirs) + assertEquals(mapOf("TP_FLAG" to "1", "TP_BARE" to null), nativeEntry.defines) + assertEquals("rebuilt", nativeEntry.status) + assertEquals( + listOf(SkippedItem("main", "app/physics/nbody.py", 30, "not pure")), + nativeEntry.skippedItems, + ) + + val mappedNative = manifest.toExtensionModuleSources() + assertEquals( + listOf( + ExtensionModuleSource( + moduleName = "app.physics.nbody", + sources = listOf(File("/abs/cache/nbody.c")), + includeDirs = listOf(File("/abs/site-packages/typedpython/runtime")), + defines = mapOf("TP_FLAG" to "1", "TP_BARE" to null), + ), + ), + mappedNative, + ) + + val mappedNotNative = manifest.toNotNativeModules() + assertEquals(2, mappedNotNative.size) + assertEquals( + NotNativeModule( + module = "app.util", + kind = NotNativeModule.Kind.NOT_MARKED, + items = emptyList(), + ), + mappedNotNative[0], + ) + assertEquals( + NotNativeModule( + module = "app.text", + kind = NotNativeModule.Kind.NOTHING_COMPILED, + items = listOf( + SkippedItem("f", "app/text.py", 2, "str slice is not supported"), + SkippedItem("g", "app/text.py", null, "no line"), + ), + ), + mappedNotNative[1], + ) + assertNull(mappedNotNative[1].items[1].line) + } + + @Test + fun AdditiveTopLevelOrEntryKeysDoNotBreakVersion1Parsing() { + val withExtraKeys = sampleManifestJson.replace( + "\"version\": 1,", + "\"version\": 1, \"futureTopKey\": 42,", + ) + val manifest = parseTypedPythonEmitCManifest(withExtraKeys) + assertEquals(1, manifest.nativeModules.size) + assertEquals(2, manifest.notNativeModules.size) + } + + @Test + fun rejectsUnknownFormatVersionOrKind() { + val wrongFormat = sampleManifestJson.replace("\"typedpython-emit-c\"", "\"other-format\"") + val formatErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongFormat) } + assertTrue("typedpython-emit-c" in formatErr.message.orEmpty()) + + val wrongVersion = sampleManifestJson.replace("\"version\": 1", "\"version\": 2") + val versionErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongVersion) } + assertTrue("version" in versionErr.message.orEmpty()) + + val wrongKind = sampleManifestJson.replace("\"NOT_MARKED\"", "\"UNKNOWN_KIND\"") + val kindErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongKind) } + assertTrue("UNKNOWN_KIND" in kindErr.message.orEmpty()) + } + + @Test + fun buildsDeterministicCommandLineWithSortedInputFiles() { + val root = File("/proj/src/main") + val cache = File("/proj/build/typedpython/c-cache") + val manifest = File("/proj/build/typedpython/emit-c-manifest.json") + val b = File("/proj/src/main/app/b.py") + val a = File("/proj/src/main/app/a.py") + + val cmd = typedPythonEmitCCommand( + pythonExecutable = "/venv/bin/python3", + files = listOf(b, a), + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + assertEquals( + listOf( + "/venv/bin/python3", + "-m", + "typedpython", + "emit-c", + a.absolutePath, + b.absolutePath, + "--root", + root.absolutePath, + "--cache", + cache.absolutePath, + "--manifest", + manifest.absolutePath, + ), + cmd, + ) + } + + @Test + fun taskDeclaresGradleInputsAndOutputsForIncrementalAndBuildCacheSupport() { + val getters = TypedPythonEmitCTask::class.java.methods.associateBy { it.name } + + val getSources = getters.getValue("getSources") + assertTrue(getSources.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getSources.isAnnotationPresent(SkipWhenEmpty::class.java)) + assertTrue(getSources.isAnnotationPresent(IgnoreEmptyDirectories::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getSources.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getRootDirectory").isAnnotationPresent(Internal::class.java)) + assertTrue(getters.getValue("getPythonExecutable").isAnnotationPresent(Input::class.java)) + + val getPythonPath = getters.getValue("getPythonPath") + assertTrue(getPythonPath.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getPythonPath.isAnnotationPresent(Optional::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getPythonPath.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getCacheDirectory").isAnnotationPresent(OutputDirectory::class.java)) + assertTrue(getters.getValue("getManifestFile").isAnnotationPresent(OutputFile::class.java)) + } + + @Test + fun taskActionExecutesStubEmitCAndParsesResultingManifest() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef inc(n: int) -> int:\n return n + 1\n") + } + val fakeStubRoot = project.file("fake-pythonpath").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + val runtimeDir = File(tpPkg, "runtime").apply { mkdirs() } + File(runtimeDir, "tp_runtime.h").writeText("/* header */\n") + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json, sys + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitTypedPythonC", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + task.emitC() + + val parsed = task.readManifest() + assertEquals(1, parsed.nativeModules.size) + val mapped = parsed.toExtensionModuleSources().single() + assertEquals("app.fast", mapped.moduleName) + assertTrue(mapped.sources.single().isFile) + assertEquals("fast.c", mapped.sources.single().name) + } + + @Test + fun taskActionFailsWithDiagnosticsOnExit1Or2() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/broken.py").apply { + parentFile.mkdirs() + writeText("def f(x): return x\n") + } + val fakeStubRoot = project.file("fake-failing-tp").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import sys + print("typedpython: broken.py does not pass the gate in compiled mode", file=sys.stderr) + sys.exit(1) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitBroken", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + val err = assertFailsWith { task.emitC() } + assertTrue("exit 1" in err.message.orEmpty(), err.message) + assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) + } + + @Test + fun pluginDoesNotRegisterEmitTypedPythonCWhenSourceDirIsUnset() { + val project = ProjectBuilder.builder().build() + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + assertEquals(null, project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK)) + } + + @Test + fun pluginRegistersEmitTypedPythonCFromExtensionAndExecutesIt() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef f(x: int) -> int:\n return x + 1\n") + } + File(pkgRoot, "app/__init__.pyi").writeText("") + val fakeStubRoot = project.file("fake-tp-plugin").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + assert all(not x.endswith(".pyi") for x in a.paths), a.paths + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + extension.typedPythonSourceDir.set(pkgRoot) + extension.typedPythonExecutable.set("python3") + extension.typedPythonPath.from(fakeStubRoot) + + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + val task = project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK) as? TypedPythonEmitCTask + ?: error("expected $EMIT_TYPEDPYTHON_C_TASK to be registered by PythonBindingsPlugin") + assertEquals(listOf(pyFile.canonicalFile), task.sources.files.map { it.canonicalFile }) + assertEquals(pkgRoot.canonicalFile, task.rootDirectory.get().asFile.canonicalFile) + assertEquals( + project.layout.buildDirectory.dir("typedpython/c-cache").get().asFile.canonicalFile, + task.cacheDirectory.get().asFile.canonicalFile, + ) + assertEquals( + project.layout.buildDirectory.file("typedpython/emit-c-manifest.json").get().asFile.canonicalFile, + task.manifestFile.get().asFile.canonicalFile, + ) + + task.emitC() + val parsed = task.readManifest() + assertEquals("app.fast", parsed.toExtensionModuleSources().single().moduleName) + } +} diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt index a2b11a193..82cca5f33 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt @@ -1,6 +1,7 @@ package python.multiplatform.gradle.stubs import python.multiplatform.gradle.model.DeclarationModel +import python.multiplatform.gradle.model.DeclaredParameter import python.multiplatform.gradle.model.KotlinTypeModel import kotlin.test.Test import kotlin.test.assertFalse @@ -60,4 +61,100 @@ class CompanionHandleStubTest { val body = stubs.getValue("androidx/compose/ui/__init__.pyi") assertFalse("receiver: _t.Any" in body, body) } + + @Test + fun classWithCompanionHandleInstanceMembersAndCompanionConstantsIsFoldedIntoClassStub() { + val m3Pkg = "androidx.compose.material3" + val designSystem = "$m3Pkg.Material3DesignSystem" + val designSystemType = KotlinTypeModel(designSystem) + val colorSchemeType = KotlinTypeModel("$m3Pkg.ColorScheme") + + val companionHandle = DeclarationModel( + simpleName = "Companion", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Companion", + isCompanionHandle = true, + ) + + val dark = DeclarationModel( + simpleName = "Dark", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Dark", + ) + + val light = DeclarationModel( + simpleName = "Light", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Light", + ) + + val method = DeclarationModel( + simpleName = "validate", + owner = designSystem, + ownerIsClass = true, + receiver = designSystemType, + receiverBoundaryTag = "OBJECT", + parameters = emptyList(), + returnType = KotlinTypeModel("kotlin.Unit"), + returnBoundaryTag = "UNIT", + kind = "METHOD", + bindingName = "$designSystem.validate", + ) + + val property = DeclarationModel( + simpleName = "colorScheme", + owner = designSystem, + ownerIsClass = true, + receiver = designSystemType, + receiverBoundaryTag = "OBJECT", + parameters = emptyList(), + returnType = colorSchemeType, + returnBoundaryTag = "OBJECT", + kind = "GETTER", + bindingName = "$designSystem.colorScheme", + ) + + val usage = DeclarationModel( + simpleName = "useSystem", + owner = m3Pkg, + ownerIsClass = false, + receiver = null, + parameters = listOf(DeclaredParameter("system", designSystemType, declaresDefault = false, boundaryTag = "OBJECT")), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "FUNCTION", + bindingName = "$m3Pkg.useSystem", + ) + + val files = renderKotlinFqnStubs(listOf(companionHandle, dark, light, method, property, usage)) + assertFalse(files.keys.any { it.startsWith("androidx/compose/material3/Material3DesignSystem") }, files.keys.toString()) + + val body = files.getValue("androidx/compose/material3/__init__.pyi") + assertTrue("class Material3DesignSystem:" in body, body) + assertTrue(" Dark: Material3DesignSystem" in body, body) + assertTrue(" Light: Material3DesignSystem" in body, body) + assertTrue(" def validate(self) -> None:" in body, body) + assertTrue(" @property\n def colorScheme(self) -> ColorScheme:" in body, body) + assertTrue("def useSystem(system: Material3DesignSystem) -> Material3DesignSystem:" in body, body) + assertFalse("system: _t.Any" in body, body) + assertFalse("-> _t.Any" in body, body) + } } diff --git a/python-multiplatform-ksp/build.gradle.kts b/python-multiplatform-ksp/build.gradle.kts index 67b3f22ac..c50576ae8 100644 --- a/python-multiplatform-ksp/build.gradle.kts +++ b/python-multiplatform-ksp/build.gradle.kts @@ -48,3 +48,5 @@ publishing { } } } + +apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) diff --git a/python-multiplatform-ksp/fixtures/app/build.gradle.kts b/python-multiplatform-ksp/fixtures/app/build.gradle.kts index 57ec527d8..d37be1eb5 100644 --- a/python-multiplatform-ksp/fixtures/app/build.gradle.kts +++ b/python-multiplatform-ksp/fixtures/app/build.gradle.kts @@ -23,10 +23,11 @@ kotlin { val downloadDir = project(":python-multiplatform").layout.buildDirectory.dir("python-standalone").get().asFile val pyVersion = project.findProperty("pythonVersion")?.toString() ?: project.rootProject.version.toString() val libVersion = pyVersion.split('.').subList(0, 2).joinToString(".") - // Version-keyed, matching python-multiplatform's `extractedDir`. - val targetExtractDir = "$downloadDir/extracted/$pyVersion/android-aarch64/prefix" + // Version-keyed, matching python-multiplatform's `extractedDir`; the free-threaded 3.15t + // tree (`android-aarch64-freethreaded`, `libpython3.15t.so`), the only Android build (#328). + val targetExtractDir = "$downloadDir/extracted/$pyVersion/android-aarch64-freethreaded/prefix" binaries.getTest(org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG).linkerOpts.addAll( - listOf("-L$targetExtractDir/lib/", "-lpython$libVersion", "-Wl,--allow-shlib-undefined"), + listOf("-L$targetExtractDir/lib/", "-lpython${libVersion}t", "-Wl,--allow-shlib-undefined"), ) } // The *second* Native leaf, and it is here for one reason: two of them make the default @@ -68,7 +69,7 @@ pythonBindings { // // - headers: `python-multiplatform`'s `cpythonIncludeDirectories` provider // (`docs/platforms/python-version-acquisition.md` §7), for this host's macOS target and the -// flavour `-PpythonFreeThreaded` selects. The provider carries the `downloadPython_*` task. +// free-threaded flavour, the only one desktop embeds (#158). The provider carries the `downloadPython_*` task. // - extension suffix: `typedpython/build_bridge.py` reads `EXT_SUFFIX` from the `_sysconfigdata_*.py` // of that same extracted tree and cross-checks `Py_GIL_DISABLED` against its `pyconfig.h`. // @@ -106,12 +107,25 @@ val typedPythonSkipReason: String? = when { else -> null } +pythonBindings { + typedPythonSourceDir.set(file("src/desktopTest/typedpython")) + typedPythonExecutable.set(typedPythonHost?.absolutePath ?: "python3") + typedPythonPath.from(rootDir.resolve("python-multiplatform-ksp/src/main/python")) +} + +val typedPythonEmitCManifest = layout.buildDirectory.file("typedpython/emit-c-manifest.json") + +tasks.matching { it.name == "emitTypedPythonC" }.configureEach { + val skip = typedPythonSkipReason + onlyIf("TypedPython emit-c: ${skip ?: "enabled"}") { skip == null } +} + @Suppress("UNCHECKED_CAST") val typedPythonIncludeDir: Provider? = typedPythonMacTarget?.let { target -> val includes = project(":python-multiplatform").extensions.getByName("cpythonIncludeDirectories") val flavourClass = includes.javaClass.classLoader.loadClass("python.multiplatform.gradle.CPythonFlavour") - val freeThreaded = (findProperty("pythonFreeThreaded") as String?)?.toBoolean() ?: false - val flavour = flavourClass.enumConstants.single { (it as Enum<*>).name == if (freeThreaded) "FREE_THREADED" else "GIL" } + // Desktop embeds only the free-threaded build (#158). + val flavour = flavourClass.enumConstants.single { (it as Enum<*>).name == "FREE_THREADED" } includes.javaClass.getMethod("includeDir", String::class.java, flavourClass) .invoke(includes, target, flavour) as Provider } @@ -156,6 +170,7 @@ val buildTypedPythonBridge by tasks.registering(Exec::class) { tasks.named("desktopTest") { dependsOn(buildTypedPythonBridge) + dependsOn("emitTypedPythonC") // The tests import tp_kotlin_bridge.py from src/; without this the interpreter leaves a // __pycache__ in the source tree. environment("PYTHONDONTWRITEBYTECODE", "1") @@ -165,10 +180,15 @@ tasks.named("desktopTest") { } if (typedPythonSkipReason == null) { inputs.dir(typedPythonBridgeDir).withPropertyName("typedPythonBridge") + inputs.file(typedPythonEmitCManifest).withPropertyName("typedPythonEmitCManifest") systemProperty( "typedpython.bridge.properties", typedPythonBridgeDir.get().file("bridge.properties").asFile.absolutePath, ) + systemProperty( + "typedpython.emitC.manifest", + typedPythonEmitCManifest.get().asFile.absolutePath, + ) } else { systemProperty("typedpython.bridge.skip", typedPythonSkipReason) } diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt index 101d0d7a6..4a45e5e62 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt @@ -291,6 +291,19 @@ class TypedPythonKotlinCallTest { ) } + @Test + fun emitTypedPythonCTaskWritesManifestForTpKotlinBridge() { + val manifestPath = System.getProperty("typedpython.emitC.manifest") + assertTrue(manifestPath != null, "system property typedpython.emitC.manifest is unset") + val manifestFile = File(manifestPath!!) + assertTrue(manifestFile.isFile, "emitTypedPythonC ran but $manifestFile does not exist") + val text = manifestFile.readText() + assertTrue("\"format\": \"typedpython-emit-c\"" in text, text) + assertTrue("\"moduleName\": \"tp_kotlin_bridge\"" in text, text) + assertTrue("tp_kotlin_bridge.c" in text, text) + assertTrue("\"notNativeModules\": []" in text, text) + } + /** * The watched reference counts once the JVM has collected the `PyObject` wrappers the binder made * and their cleaners have handed the references back. diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py index 7677eca9c..311eb2f5b 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py @@ -1,4 +1,3 @@ -# typedpython: compiled """Issue #45: compiled TypedPython code calling Kotlin through the binder. Every name imported below is a binder proxy (`fixture/app/TypedPythonBridge.kt`, published by @@ -21,6 +20,7 @@ from fixture.app import TpSample, tpAccumulate, tpCheck, tpTouch +@compiled def drive(n: int, scale: float, token: object) -> float: """Per step: a Kotlin constructor call that returns a typed proxy, a method call on that proxy, the proxy handed back to a Kotlin function, and a PyObject handed to Kotlin -- inside native @@ -35,6 +35,7 @@ def drive(n: int, scale: float, token: object) -> float: return total +@compiled def checked(n: int, limit: float) -> float: """`tpCheck` throws in Kotlin once `acc` passes `limit`; the caller sees what Python sees.""" acc: float = 0.0 diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index d5fcbf41c..1f50ef871 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -4,11 +4,17 @@ [project] name = "typedpython" version = "0.1.0" +description = "TypedPython: statically checked Python, with marked functions compiled to CPython extension modules" +license = "Apache-2.0" +authors = [{ name = "thisisthepy" }] requires-python = ">=3.13" # Pinned exactly: the gate reads Pyrefly's per-expression type report (--report-pysa), whose # format is versioned by Pyrefly, not by a public API. dependencies = ["pyrefly==1.3.2"] +[project.urls] +Homepage = "https://github.com/thisisthepy/python-multiplatform" + [project.scripts] typedpython = "typedpython.__main__:cli" @@ -32,5 +38,5 @@ pythonpath = ["src/main/python"] addopts = "--basetemp=.pytest_tmp" markers = [ "measurement: records cost; asserts only sanity bounds", - "race_mutant: shows a concurrency test catches a mutant; depends on the race firing, so local only (not in CI)", + "race_mutant: shows a concurrency test catches a mutant; depends on the race firing, so not in push CI; run with the TypedPython CI dispatch input race_mutant (#394)", ] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index bd220e713..f368b72c0 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -1,4 +1,5 @@ """`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` +`python -m typedpython emit-c PATHS... --root DIR [--cache DIR] --manifest FILE` Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. """ @@ -36,8 +37,17 @@ def main(argv: list[str]) -> int: help="incremental cache directory (default: build/typedpython-cache)") build.add_argument("--out", type=Path, default=None, help="copy the built extensions here (default: leave them in the cache)") + emit = commands.add_parser( + "emit-c", help="generate C per module (cached) and a manifest for pypackpack's compile slot") + emit.add_argument("paths", nargs="+", type=Path, help="files, or directories to scan recursively") + emit.add_argument("--root", type=Path, required=True, help="directory module names are relative to") + emit.add_argument("--cache", type=Path, default=Path("build/typedpython-c-cache"), + help="source -> C cache directory (default: build/typedpython-c-cache)") + emit.add_argument("--manifest", type=Path, required=True, help="the JSON manifest to write") args = parser.parse_args(argv) + if args.command == "emit-c": + return _emit_c(args.paths, args.root, args.cache, args.manifest) if args.command == "build": return _build(args.paths, args.cache, args.out) if args.command == "demo": @@ -84,7 +94,7 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: print(f"{m.status:7} {m.name}: {what}{why}") rel = m.path.relative_to(root).as_posix() if m.kind == "not_marked": - print(f" {rel}: not marked for compilation (no `# typedpython: compiled`, no @compiled)") + print(f" {rel}: not marked for compilation (no @compiled)") for item in m.skipped_items: where = f"{item.file}:{item.line}" if item.line is not None else item.file print(f" {where}: {item.name}: {item.message}") @@ -94,6 +104,57 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: return EXIT_CLEAN +MANIFEST_FORMAT, MANIFEST_VERSION = "typedpython-emit-c", 1 + + +def _emit_c(paths: list[Path], root: Path, cache: Path, manifest: Path) -> int: + """SPEC N-10 / #25 step 1; the manifest format is docs/design/typedpython.md section 4.3.2a.""" + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to emit", file=sys.stderr) + return EXIT_FAILURE + try: + result = pipeline.generate_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + root = root.resolve() + runtime = Path(result.runtime_include).resolve().as_posix() + + def items(m: "pipeline.ModuleResult") -> list[dict]: + return [{"name": i.name, "file": i.file, "line": i.line, "message": i.message} + for i in m.skipped_items] + + native, other = [], [] + for m in result.modules: + if m.path.name == "__init__.py": + continue # never listed (pypackpack compile-slot.md) + rel = m.path.resolve().relative_to(root).as_posix() + if m.kind == "compiled": + assert m.c_source is not None + native.append({"moduleName": m.name, "file": rel, + "sources": [Path(m.c_source).resolve().as_posix()], + "includeDirs": [runtime], "defines": {}, "status": m.status, + "skippedItems": items(m)}) + else: + other.append({"module": m.name, "file": rel, + "kind": "NOT_MARKED" if m.kind == "not_marked" else "NOTHING_COMPILED", + "items": items(m)}) + doc = {"format": MANIFEST_FORMAT, "version": MANIFEST_VERSION, + "compilerVersion": pipeline.COMPILER_VERSION, "projectRoot": root.as_posix(), + "runtimeInclude": runtime, "nativeModules": native, "notNativeModules": other} + manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = manifest.with_name(manifest.name + ".tmp") + tmp.write_text(json.dumps(doc, indent=2) + "\n") + tmp.replace(manifest) + return EXIT_CLEAN + + def _demo(module: Path, module_argv: list[str], out: Path) -> int: import contextlib import importlib.util @@ -112,6 +173,8 @@ def _demo(module: Path, module_argv: list[str], out: Path) -> int: return EXIT_FAILURE def interpreted() -> object: + import builtins # the module's `@compiled` is a builtin name; typedpython_builtins.pth installs it + builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi index f773368b9..122e4cdf5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi +++ b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi @@ -1,3 +1,9 @@ # TypedPython's builtin names (docs/design/typedpython.md §2.3). A `__builtins__.pyi` on the type # checker's search path declares names the interpreter provides without an import. +from collections.abc import Callable +from typing import overload + +@overload def compiled[T](f: T, /) -> T: ... +@overload +def compiled[T](*, parallel: bool = ..., fastmath: bool = ...) -> Callable[[T], T]: ... diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py index d12ecea39..2f30a9490 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py @@ -164,7 +164,7 @@ def _find_vcvars() -> Path: f"with C++' workload ({MSVC_COMPONENT}), or run from a 'x64 Native Tools' prompt.") run = subprocess.run( [str(vswhere), "-latest", "-products", "*", "-requires", MSVC_COMPONENT, - "-property", "installationPath"], capture_output=True, text=True) + "-property", "installationPath"], capture_output=True, encoding="oem", errors="replace") root = run.stdout.strip().splitlines()[0].strip() if run.returncode == 0 and run.stdout.strip() else "" if not root: raise CBuildError( @@ -229,7 +229,8 @@ def build(c_source: str, module_name: str, out_dir: Path, runtime_dir: Path | No env = msvc_env() cmd = msvc_command(_msvc_cl(env), c_file, so, include, runtime, flags, gil_disabled=free_threaded(ext_suffix)) - run = subprocess.run(cmd, capture_output=True, text=True, env=env) + # cl.exe writes in the console code page (CP949 on Korean Windows), not UTF-8 + run = subprocess.run(cmd, capture_output=True, encoding="oem", errors="replace", env=env) if run.returncode != 0 or not so.is_file(): # cl.exe reports its errors on stdout raise CBuildError(f"C compiler failed ({run.returncode}): {subprocess.list2cmdline(cmd)}\n" diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index fb776ce01..d00f43fc7 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -29,7 +29,10 @@ copy-in to after the last write-back: `tp_cs_end` is part of the same `exits` that run the write-back on every path (ok, error, deopt, the last before the interpreted redo), and is a no-op on a path that left before locking. More than two list parameters are refused here (`CGenError`), as in the -front end and the verifier: CPython has no three-object critical section. A function that holds an +front end and the verifier: CPython has no three-object critical section; except in a pure function +(#255, `ir.direct_lists`): its lists are not copied, every Index is `tp_list_ld_*` +(`PyList_GetItemRef` in program order, an element of another type deopts), every Len the current +length, and no lock is taken. A function that holds an array does not poll the eval breaker (#141, open decision #152), so a long compiled array loop delays other threads' stop-the-world requests until it returns: a liveness delay, results unaffected. The `__typedpython_deopts__` read-add-write of `tp_cg_redo` runs in the module dict's @@ -57,7 +60,10 @@ Depth (issue #57): every impl function calls `tp_enter_call()` before it owns anything (a refusal returns -1 directly: RecursionError set, nothing counted) and `tp_leave_call()` in the one exit -block, so ok, error and deopt exits all leave exactly once. +block, so ok, error and deopt exits all leave exactly once. The wrapper brackets its impl call with +`tp_depth_seg_begin` / `tp_depth_seg_end` (#125): the Python depth is measured at the first check of +that call and reused by every later check until a wrapper of this module returns inside it +(tp_runtime.h "Segments"); the recursion limit and the C-stack guard are still read per frame. Fast entry (issue #147). A *bounded* function (`_bounded_functions`: no call cycle, every callee bounded, no node that can run Python code) has a call tree of at most k compiled frames. Its @@ -78,12 +84,23 @@ 2. every OBJ-typed expression evaluates into a fresh temporary that owns a new reference; temporaries are NULL at every statement boundary; 3. a temporary is consumed exactly once: moved into a local (the local's old reference is - released), moved into `*tp_out` by Return, or released (`tp_release`) right after the helper - that borrowed it returns; + released), moved into `*tp_out` by Return, moved into the slots of a fresh object by + `tp_new_fixed_move` (then emptied, `_moved_temps`), or released (`tp_release`) right after the + helper that borrowed it returns; + 3a. an operand that a helper only borrows for its own duration, the two sides of Is, the object of + a FieldGet or IsExact and the OBJ arguments of a C to C Call, names a plain OBJ local itself + (`l_x`, or `Py_None` for the None constant) instead of a copy: no Py_NewRef, no release + (`_borrow_operand`, `_operand_release`). The local's own reference keeps the object alive for + the helper's whole duration, whatever other threads do (it is this frame's reference), and a + local changes only at a statement boundary (the IR has no assignment expression), so reading + the name when the helper runs is reading it when the operand was evaluated. CPython 3.14+ does + the same for these operands (LOAD_FAST_BORROW). Issue #125; + design 4.3.4 "참조 옮기기와 빌린 피연산자"; 4. every exit of the impl, return, error, deopt, goes through one exit block that releases every OBJ local and temporary. The generated code never decrements a reference except through `tp_release` (API.md); the only - increment is `Py_NewRef` when an OBJ local is read or an OBJ parameter is adopted. + increment is `Py_NewRef` when an OBJ local is read as a value (not as a borrowed operand, 3a) or + an OBJ parameter is adopted. Local arrays (ir.NewArray / CopyArray, `Function` docstring "Local arrays"). A local of an array type is a `tp_i64_array` / `tp_f64_array` struct held by value in the impl, zero-initialised at entry @@ -114,7 +131,8 @@ impure functions (whose impl may never return 1), the impl's copy serves C to C Calls, where a failure is a deopt, so a Call of a function with a `Param.cls` parameter counts as a node that can deopt (`_can_deopt`), exactly like a callee with entry_globals; - 3. FieldGet / FieldSet / New go through `tp_field_get` / `tp_field_set` / `tp_new_fixed` with the + 3. FieldGet / FieldSet / New go through `tp_field_get` / `tp_field_set` / `tp_new_fixed_move` + (tp_new_fixed's stores, with the argument temporaries' references moved in, #125) with the captured descriptors; CheckExact deopts (return 1) on mismatch and is legal only in a pure function; Is is a plain BOOL. @@ -446,6 +464,20 @@ def _class_current_expr(obj_c: str, k: int, fields_c: str, n: int, descrs_c: str f"{fields_c}, {n}, {descrs_c})") +def _moved_temps(items: list[str]) -> str: + """After `tp_new_fixed_move` succeeded: the object owns the argument temporaries' references, so + the temporaries are emptied instead of released (module docstring, rule 3; #125). The single + mutation point of the move: returning "" releases them anyway and must fail the tests.""" + return " ".join(f"{t} = NULL;" for t in items) + + +def _operand_release(name: str, owned: bool) -> str | None: + """The temporary to release after a helper borrowed operand `name`: an owned temporary, never a + borrowed OBJ local (rule 3a; #125). The single mutation point of the borrow: releasing every + operand drops a reference the local still owns and must fail the tests.""" + return name if owned else None + + def _watch_event_stmt() -> str: """The body of the module's dict-watcher callback (module docstring, rule 5).""" return "return tp_globals_watch_event(event, key, tp_watch_names);" @@ -494,10 +526,19 @@ def _runs_no_python(f: ir.Function) -> bool: def _bounded_functions(functions: dict[str, ir.Function]) -> dict[str, int]: """name -> k for every bounded function (module docstring, "Fast entry"): in no call cycle, every callee bounded, `_runs_no_python`. k(f) = 1 + max(k(callee)), 1 without callees: the most - compiled frames a call of `f` can have in flight at once.""" + compiled frames a call of `f` can have in flight at once. + A `fastmath` function is never given an `inline` fast twin (`tp_fimpl_*`) when some other + function in the module calls it without `fastmath`, so a non-fastmath caller can never inline + a fastmath body (#332).""" callees: dict[str, set[str]] = {} for name, f in functions.items(): callees[name] = {e.function for e in _all_exprs(f) if isinstance(e, ir.Call)} + called_from_exact = { + c + for caller_name, cs in callees.items() + if not functions[caller_name].fastmath + for c in cs + } k: dict[str, int | None] = {} # None: not bounded (also while on the DFS stack) def visit(name: str) -> int | None: @@ -505,7 +546,7 @@ def visit(name: str) -> int | None: return k[name] k[name] = None # on the stack: reaching it again is a cycle f = functions[name] - if not _runs_no_python(f): + if not _runs_no_python(f) or (f.fastmath and name in called_from_exact): return None deepest = 0 for c in callees[name]: @@ -743,6 +784,27 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#if !defined(_MSC_VER) || defined(__clang__) /* cl.exe warns on an unknown pragma */") w("#pragma STDC FP_CONTRACT OFF") w("#endif") + if any(f.fastmath for f in module.functions): + w("/* SPEC N-8 (#153, #332): per-function `@compiled(fastmath=True)` opt-in (NOT CPython-identical). */") + w("#if defined(__clang__)") + w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") + w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") + w("#elif defined(__GNUC__)") + # The same scope as clang (#368): reassociation and FMA contraction only. + # associative-math takes effect only with no-signed-zeros and no-trapping-math; never + # fast-math, whose finite-math-only folds `x != x` and isnan away. + w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(" + "\\\"associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast\\\")\")") + w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") + w("#elif defined(_MSC_VER)") + # cl.exe has no reassociation-only switch: float_control(precise, off) is /fp:fast, which also + # folds `x != x` away (Windows CI, #368). So MSVC gets FMA contraction only. + w("#define TP_FASTMATH_BEGIN __pragma(fp_contract(on))") + w("#define TP_FASTMATH_END __pragma(fp_contract(off))") + w("#else") + w("#define TP_FASTMATH_BEGIN") + w("#define TP_FASTMATH_END") + w("#endif") w("") w(f"static const char tp_source[] =\n {c_source_init(source)};") w(f"static const char tp_source_path[] = {c_string(display_path)};") @@ -891,7 +953,9 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: if f.entry_globals: _check_entry_globals(f, functions) n_arrays = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params) - if n_arrays > 2: + if ir.direct_lists(f): + _check_direct_lists(f) + elif n_arrays > 2: raise CGenError(f"{f.name}: {n_arrays} array parameters; at most two (CPython offers one- and " "two-object critical sections only, and a third lock could suspend the others)") for p in f.params: @@ -929,6 +993,26 @@ def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> return True +def _check_direct_lists(f: ir.Function) -> None: + """#255 (ir.direct_lists): what a pure function with three or more lists read element by element + may not have.""" + why = None + if not f.may_deopt: + why = "may_deopt is False (every element read can deopt)" + elif f.entry_globals: + why = f"entry global {f.entry_globals[0].name} (it would be read once at entry)" + elif any(isinstance(p, ir.ArrayParam) and (p.buffer is not None or p.stored + or p.type not in _ARRAY_PFX) for p in f.params): + why = "a buffer parameter, a stored list or a list of another element type" + elif any(isinstance(e, ir.CopyArray) and any(q.name == e.src for q in f.params) for e in _all_exprs(f)): + why = "a CopyArray of a list parameter" + elif any(isinstance(s, ir.ForRange) and s.par is not None for s in _walk_stmts(f.body)): + why = "a parallel loop" + if why is not None: + raise CGenError(f"{f.name}: three or more list parameters are read element by element (#255), " + f"which rules out {why}") + + def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> None: names = [g.name for g in f.entry_globals] if len(set(names)) != len(names): @@ -1048,6 +1132,9 @@ def __init__(self, mg: _ModGen, f: ir.Function, fast: bool = False, rmode: str | self.depth = 1 self.temps: list[tuple[str, str]] = [] # (C type, name) self.arrays = {p.name: p for p in f.params if isinstance(p, ir.ArrayParam)} + # #255: a pure function with three or more lists reads them element by element (no copy-in) + self.direct = {p.name for p in f.params if isinstance(p, ir.ArrayParam) and p.buffer is None} \ + if ir.direct_lists(f) else set() self.par_defs: list[str] = [] # outlined region bodies (SPEC N-15), emitted before the impl self.local_arrays: dict[str, Type] = {} # name -> F64_ARRAY / I64_ARRAY self.brk: list = [] # per open loop: goto label for `break`, or None @@ -1463,15 +1550,18 @@ def ev_Call(self, e: ir.Call) -> str | None: int_arg = a.type is Type.INT and p.type is Type.I64 and e.redo if a.type is not p.type and not int_arg: raise CGenError(f"{self.f.name}: Call {e.function} arg {p.name}: {a.type} for {p.type}") - v = self.ev(a) + if a.type is Type.OBJ: + vo = self._borrow_operand(a, "Call argument") # rule 3a + v = vo[0] + owned.append(vo) + else: + v = self.ev(a) values.append(v) if int_arg: args.append(f"{v}.v") int_args.append(v) else: args.append(v) - if a.type is Type.OBJ: - owned.append(v) t = ti = None self.emit("tp_s = 0;") if not self.f.pure and not self.fast: @@ -1512,7 +1602,7 @@ def ev_Call(self, e: ir.Call) -> str | None: # a pure callee using classes returns 1 when a class check fails (module docstring, # rule 7): redo it interpreted; this impure caller must not deopt self._redo_call(callee, e.args, values, t) - self._own_release(*owned) + self._release_operands(*owned) for v in int_args: self.emit(f"tp_int_release(&{v});") self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") @@ -1602,17 +1692,35 @@ def _obj_operand(self, e: ir.Expr, what: str) -> str: raise CGenError(f"{self.f.name}: {what} on {e.type}") return self.ev(e) + def _borrow_operand(self, e: ir.Expr, what: str) -> tuple[str, bool]: + """An OBJ operand the helper only borrows (module docstring, rule 3a): (C name, owned). + A plain OBJ local (not virtual, not a region pointer) names itself and the None constant + names Py_None, both borrowed; anything else is an owned temporary as `_obj_operand`.""" + if e.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: {what} on {e.type}") + if not self.rmode: + if (isinstance(e, ir.Local) and e.name not in self.virtual + and self.types.get(e.name) is Type.OBJ): + return _ident("l", e.name), False + if isinstance(e, ir.Const) and e.value is None: + return "Py_None", False + return self.ev(e), True + + def _release_operands(self, *ops: tuple[str, bool]) -> None: + self._own_release(*[n for n in (_operand_release(name, owned) for name, owned in ops) + if n is not None]) + def ev_Is(self, e: ir.Is) -> str: if self.rmode: a, b = self._r_ptr_name(e.left), self._r_ptr_name(e.right) t = self.tmp("int") self.emit(f"{t} = ({a} {'!=' if e.negate else '=='} {b});") return t - a = self._obj_operand(e.left, "Is") - b = self._obj_operand(e.right, "Is") + ao = self._borrow_operand(e.left, "Is") + bo = self._borrow_operand(e.right, "Is") t = self.tmp("int") - self.emit(f"{t} = ({a} {'!=' if e.negate else '=='} {b});") - self._own_release(a, b) + self.emit(f"{t} = ({ao[0]} {'!=' if e.negate else '=='} {bo[0]});") + self._release_operands(ao, bo) return t def _current(self, obj_c: str, cls: str) -> str: @@ -1639,7 +1747,8 @@ def ev_IsExact(self, e: ir.IsExact) -> str: self.emit(f"{t} = ({p} != NULL);") return t k = self._class(e.cls) - o = self._obj_operand(e.obj, "IsExact") + oo = self._borrow_operand(e.obj, "IsExact") + o = oo[0] t = self.tmp("int") self.emit(f"tp_s = {self._global_ok(e.cls)};") self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") @@ -1654,7 +1763,7 @@ def ev_IsExact(self, e: ir.IsExact) -> str: self._own_release(g) self.depth -= 1 self.emit("}") - self._own_release(o) + self._release_operands(oo) return t def ev_CheckExact(self, e: ir.CheckExact) -> str: @@ -1686,7 +1795,8 @@ def ev_FieldGet(self, e: ir.FieldGet) -> str: raise CGenError(f"{self.f.name}: FieldGet {e.cls}.{e.field} on virtual {e.obj.name}") return self._own_copy(_ident("l", _vfield(e.obj.name, cd.fields.index(e.field)))) slot = self.mg.slot(self.f.name, e.cls, e.field) - o = self._obj_operand(e.obj, "FieldGet") + oo = self._borrow_operand(e.obj, "FieldGet") + o = oo[0] t = self.tmp("PyObject *") self.emit(f"tp_s = {self._current(o, e.cls)};") self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") @@ -1699,7 +1809,7 @@ def ev_FieldGet(self, e: ir.FieldGet) -> str: self.emit(f"{t} = tp_getattr({o}, tp_st->names[{self.mg.name_index(e.field)}]);") self.depth -= 1 self.emit("}") - self._own_release(o) + self._release_operands(oo) self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") return t @@ -1753,8 +1863,10 @@ def ev_New(self, e: ir.New) -> str: self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") self.emit(av) self.emit(" if (tp_s) {") - self.emit(f" {t} = tp_new_fixed((PyTypeObject *)tp_st->classes[{k}], " + self.emit(f" {t} = tp_new_fixed_move((PyTypeObject *)tp_st->classes[{k}], " f"{f'&tp_st->slots[{base}]' if items else 'NULL'}, tp_av, {len(items)});") + if items: # the object owns them now (rule 3) + self.emit(f" if ({t} != NULL) {{ {_moved_temps(items)} }}") self.emit(" } else {") self.depth += 1 if self.f.pure: @@ -1766,7 +1878,7 @@ def ev_New(self, e: ir.New) -> str: self.emit(f" {t} = tp_call({gk} ? tp_st->classes[{k}] : {g}, tp_av, {len(items)}, NULL);") self.depth -= 1 self.emit(" } }") - self._own_release(*items) # the slots took their own references + self._own_release(*items) # emptied when moved into the slots; released on the slow path if g is not None: self._own_release(g) self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") @@ -1928,6 +2040,9 @@ def _array(self, name: str) -> tuple[Type, str]: def ev_Len(self, e: ir.Len) -> str: _, ptr = self._array(e.array) t = self.tmp("int64_t") + if e.array in self.direct: + self.emit(f"{t} = (int64_t)PyList_GET_SIZE({ptr}->list); /* #255: the current length */") + return t self.emit(f"{t} = (int64_t){ptr}->len;") return t @@ -1954,6 +2069,17 @@ def ev_Index(self, e: ir.Index) -> str: at, ptr = self._array(e.array) if e.type is not _ARRAY_ELEM[at]: raise CGenError(f"{self.f.name}: Index of {at} typed {e.type}") + if e.array in self.direct: + # #255: one PyList_GetItemRef (the interpreter's read), in program order; IndexError, or a + # deopt for an element of another type + if e.index.type is not Type.I64: + raise CGenError(f"{self.f.name}: index of type {e.index.type}") + i = self.ev(e.index) + t = self.tmp(_CT[e.type]) + sfx = "f64" if at is Type.F64_ARRAY else "i64" + self.emit(f"tp_s = tp_list_ld_{sfx}({ptr}->list, {i}, &{t});") + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t buffer = self._buffer(e.array) slot = self._slot(at, ptr, e.index, e.proven, 0, buffer) t = self.tmp(_CT[e.type]) @@ -2211,7 +2337,8 @@ def st_If(self, s: ir.If) -> None: @property def snapshot(self) -> bool: """Holds an array copy-in or an entry-globals snapshot, so must not run user code (#141).""" - return bool(self.f.entry_globals) or any(isinstance(p, ir.ArrayParam) for p in self.f.params) + return bool(self.f.entry_globals) or any(isinstance(p, ir.ArrayParam) and p.name not in self.direct + for p in self.f.params) def poll(self, every_time: bool = False) -> None: """At the top of each loop iteration (#141): signals, pending calls, GIL hand-off. @@ -2892,7 +3019,8 @@ def generate(self) -> str: """The impl (preceded by its fast twin when bounded) and the wrapper.""" parts = [] if self.rmode: - return self.impl() + text = self.impl() + return f"TP_FASTMATH_BEGIN\n{text}TP_FASTMATH_END\n" if self.f.fastmath else text if self.f.name in self.mg.bounded: twin = _FnGen(self.mg, self.f, fast=True) twin_text = twin.impl() @@ -2901,6 +3029,8 @@ def generate(self) -> str: impl_text = self.impl() parts += self.par_defs parts.append(impl_text) + if self.f.fastmath: + parts = ["TP_FASTMATH_BEGIN", *parts, "TP_FASTMATH_END"] parts.append(self.wrapper()) return "\n".join(parts) @@ -2919,13 +3049,15 @@ def wrapper(self) -> str: args = [] arrays = [] # list parameters: copy-in / write-back under a critical section buffers = [] # buffer parameters: a Py_buffer export held for the whole call (#149) - if any(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params): + direct = [] # #255: lists read element by element (ir.direct_lists), no copy-in, no lock + is_direct = ir.direct_lists(f) + if any(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params) and not is_direct: L.append(" tp_cs tp_lock = {0};") for i, p in enumerate(f.params): if isinstance(p, ir.ArrayParam): L.append(f" {'tp_buf' if p.buffer else _ARRAY_CT[p.type]} tp_a{i} = {{0}};") args.append(f"&tp_a{i}") - (buffers if p.buffer else arrays).append((i, p)) + (buffers if p.buffer else direct if is_direct else arrays).append((i, p)) elif p.type is Type.OBJ: args.append(f"tp_args[{i}]") # borrowed for the call's duration else: @@ -2984,6 +3116,13 @@ def wrapper(self) -> str: f"{fmt1}, {fmt2}, {int(p.stored)}, &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") L.append(" if (tp_s != 0) goto tp_deopt;") + if direct: + # #255: exact lists (a subclass may override __getitem__ / __len__), not copied; the + # wrapper's arguments keep them alive for the call + L.append(" if (" + " || ".join(f"!PyList_CheckExact(tp_args[{i}])" for i, _ in direct) + + ") goto tp_deopt;") + for i, _ in direct: + L.append(f" tp_a{i}.list = tp_args[{i}]; /* borrowed; read by tp_list_ld_* */") # aliasing guard, then copy-in if len(arrays) > 1: objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) @@ -3029,7 +3168,10 @@ def wrapper(self) -> str: lock = "&tp_lock" if lst else "NULL" L.append(f" tp_osr tp_ox = {{{lock}, {{{l0}, {l1}}}, NULL}};") L.append(" tp_t->osr = &tp_ox; /* #152: taken by the impl at entry */") + # #125: the impl call is one depth segment (tp_runtime.h "Segments"): reset before and after + L.append(" tp_depth_seg_begin(tp_t);") L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module, tp_ts, tp_t{''.join(', ' + a for a in args)});") + L.append(" tp_depth_seg_end(tp_t);") if osr: L.append(" tp_t->osr = NULL;") L.append(" if (tp_s == 2) { /* left for the continuation: its result, lists already final */") @@ -3548,6 +3690,11 @@ def _exec_function(mg: _ModGen) -> str: " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_fast_hits__\", tp_cf) < 0) goto tp_done;", " tp_release(&tp_cf);", "#endif", + "#ifdef TP_TRACE_DEPTH", + " tp_cf = PyCFunction_NewEx(&tp_trace_frame_reads_def, tp_module, NULL);", + " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_frame_reads__\", tp_cf) < 0) goto tp_done;", + " tp_release(&tp_cf);", + "#endif", # SPEC N-14: strong references to the originals' code objects, read by tp_traced at every entry f" for (tp_k = 0; tp_k < {len(mg.names)}; tp_k++) {{", " tp_k2 = PyDict_GetItemRef(tp_table, tp_st->names[tp_k], &tp_f);", diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 0da894f10..1c3c2332e 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -3,8 +3,7 @@ `lower(path)` reads one module and returns an `ir.Module`: - Which functions: module-level `def`s decorated with the builtin `@compiled` (a bare name, no - import, #42), or every module-level `def` when the first non-empty line is - `# typedpython: compiled`. + import, #42). That decorator is the only mark; there is no module-level pragma (#353). - A function is lowered only when **all** of it is expressible in the IR; otherwise it is left out and `Module.skipped[name]` says why, with the line. There is no partial lowering. - Types: parameters and returns from their annotations (`int`/`float`/`bool`, `list[float]` and @@ -80,8 +79,6 @@ from typedpython import gate, ir, pyrefly, rebinding, resume, taint from typedpython.ir import Type -MARKER = "# typedpython: compiled" - I64_MIN, I64_MAX = -2**63, 2**63 - 1 TOP = (I64_MIN, I64_MAX) EXACT_INT_FLOAT = 2**53 # |n| <= this: int/int true division is exact-rounded in C (ir.BinOp) @@ -193,7 +190,9 @@ def lower(path: Path) -> ir.Module: info.classes = class_decls skipped.update(class_skips) signatures: dict[str, _Signature] = {} - for node, reason in _targets(tree, source, info): + marked: set[str] = set() + fastmath_marked: set[str] = set() + for node, reason, parallel, fastmath in _targets(tree, source, info): if reason is not None: skipped[node.name] = reason continue @@ -202,6 +201,11 @@ def lower(path: Path) -> ir.Module: signatures[node.name] = _signature(node, info.classes, info) except Skip as e: skipped[node.name] = e.reason + else: + if parallel: + marked.add(node.name) + if fastmath: + fastmath_marked.add(node.name) for owner, node, reason in _method_targets(tree, info): q = f"{owner}.{node.name}" if reason is not None: @@ -239,28 +243,14 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - marked = _parallel_marked(tree, source) - functions = [dataclasses.replace(f, parallel=True) if f.name in marked else f for f in functions] + functions = [ + dataclasses.replace(f, parallel=f.name in marked, fastmath=f.name in fastmath_marked) + if (f.name in marked or f.name in fastmath_marked) else f + for f in functions + ] return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) -PARALLEL_MARKER = "# typedpython: parallel" - - -def _parallel_marked(tree: ast.Module, source: str) -> set[str]: - """Module-level functions the user marked for SPEC N-15: the comment line `# typedpython: parallel` - directly above the `def` (above its decorators if it has any). A comment, like the module marker, - so the interpreted module is unchanged and needs no runtime name.""" - lines = source.splitlines() - out: set[str] = set() - for node in tree.body: - if isinstance(node, ast.FunctionDef): - first = min([node.lineno] + [d.lineno for d in node.decorator_list]) - if first >= 2 and lines[first - 2].strip() == PARALLEL_MARKER: - out.add(node.name) - return out - - def _module_info(tree: ast.Module) -> _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] = {} @@ -434,34 +424,62 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") -def _whole_module(source: str) -> bool: - return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER +def _is_compiled_decorator(d: ast.expr) -> bool: + return (isinstance(d, ast.Name) and d.id == "compiled") or ( + isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "compiled" + ) + + +def _compiled_decorator_options(d: ast.expr) -> tuple[bool, bool, str | None]: + """Parse `@compiled` or `@compiled(parallel=..., fastmath=...)` -> (parallel, fastmath, error_reason_or_None).""" + if isinstance(d, ast.Name): + return False, False, None + assert isinstance(d, ast.Call) + if d.args: + return False, False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " + f"(got positional `{_src(d.args[0])}`)") + parallel = False + fastmath = False + seen: set[str] = set() + for kw in d.keywords: + if kw.arg is None: + return False, False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" + if kw.arg not in ("parallel", "fastmath"): + return False, False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " + "supported (only `parallel` and `fastmath` with literal `True` or `False`)") + if kw.arg in seen: + return False, False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" + seen.add(kw.arg) + if not (isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, bool)): + return False, False, (f"line {d.lineno}: `@compiled({kw.arg}=...)` requires a literal `True` " + f"or `False`, got `{_src(kw.value)}`") + if kw.arg == "parallel": + parallel = kw.value.value + else: + fastmath = kw.value.value + return parallel, fastmath, None def is_opted_in(path: Path) -> bool: - """Whether the front end considers the module for compilation at all: the `# typedpython: compiled` - marker as its first non-empty line, or a module-level `def` / `class` decorated `@compiled` - (the same tests `_targets` and `_classes` apply).""" + """Whether the front end considers the module for compilation at all: a module-level `def` / `class` + decorated `@compiled` (the same test `_targets` and `_classes` apply).""" source = Path(path).read_text() - if _whole_module(source): - return True return any( isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) - and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) + and any(_is_compiled_decorator(d) for d in node.decorator_list) for node in ast.parse(source, filename=str(path)).body) def _classes( tree: ast.Module, source: str, info: _ModuleInfo, ) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]: - whole = _whole_module(source) decls: dict[str, ir.ClassDecl] = {} skipped: dict[str, str] = {} for node in tree.body: if not isinstance(node, ast.ClassDef): continue marked = any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) - if not (whole or marked): + if not marked: continue result = _class_decl(node, tree, info) if isinstance(result, str): @@ -607,16 +625,17 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, -) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - whole_module = _whole_module(source) - found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]]: + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue - marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] - if not (whole_module or marked): + marked = [d for d in node.decorator_list if _is_compiled_decorator(d)] + if not marked: continue reason = None + parallel = False + fastmath = False if marked and info.bound("compiled"): line = info.bindings["compiled"][0][0] reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " @@ -624,12 +643,21 @@ def _targets( elif other := [d for d in node.decorator_list if d not in marked]: reason = (f"line {other[0].lineno}: decorator `@{_src(other[0])}` is not supported " "(only `@compiled`)") - elif isinstance(node, ast.AsyncFunctionDef): - reason = f"line {node.lineno}: async function is not supported" - elif len(info.bindings.get(node.name, [])) > 1: - lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) - reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" - found.append((node, reason)) + else: + for d in marked: + par, fm, r = _compiled_decorator_options(d) + if r is not None: + reason = r + break + parallel = parallel or par + fastmath = fastmath or fm + if reason is None: + if isinstance(node, ast.AsyncFunctionDef): + reason = f"line {node.lineno}: async function is not supported" + elif len(info.bindings.get(node.name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) + reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" + found.append((node, reason, parallel, fastmath)) return found @@ -715,13 +743,6 @@ def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = No params.append(ir.Param(a.arg, Type.OBJ, c[0], c[1])) else: params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) - n_lists = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in params) - if n_lists > 2: - # Free-threaded CPython (#159): the lists are locked from copy-in to write-back, and CPython - # offers one- and two-object critical sections only (a nested third could suspend the others). - raise Skip(line, f"more than two list parameters ({n_lists}) are not supported: compiled code " - "holds each list's lock from copy-in to write-back, and free-threaded CPython " - "has no lock for more than two objects at once") if fn.returns is None: raise Skip(line, "no return annotation") if isinstance(fn.returns, ast.Constant) and fn.returns.value is None: @@ -1050,6 +1071,22 @@ def run(self) -> ir.Function: pure = not self.impure assert pure or not self.deopts or not self.impure_mode, \ "impure mode emitted a deopting node" + n_lists = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in params) + if n_lists > 2: + # Free-threaded CPython (#159): an impure function's lists are locked from copy-in to + # write-back, and CPython offers one- and two-object critical sections only (a nested + # third could suspend the others). A pure one reads them element by element (#255, + # ir.direct_lists), and a read of an element of another type deopts. + if not pure: + raise Skip(self.fn.lineno, f"more than two list parameters ({n_lists}) are not supported " + "in a function with an effect: compiled code holds each list's " + "lock from copy-in to write-back, and free-threaded CPython has " + "no lock for more than two objects at once") + if self.entry_used: + raise Skip(self.fn.lineno, f"more than two list parameters ({n_lists}) and a module global " + "read: the lists are read element by element, so the global " + "cannot be read once at entry (#255)") + self.deopts = True fn = ir.Function( name=self.sig.qname, params=params, returns=self.sig.returns, locals=locals_, body=body, pure=pure, may_deopt=self.deopts, source_line=self.fn.lineno, @@ -2499,4 +2536,4 @@ def _breaks(body: tuple[ir.Stmt, ...]) -> bool: return False -__all__ = ["lower", "MARKER"] +__all__ = ["lower"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 368c1c3ba..f29cc38be 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -817,9 +817,12 @@ class Function: # rejects a function whose declared entries differ (`verify/virtual-escape`). The front end # leaves it empty. virtual: tuple[str, ...] = () - # SPEC N-15: the user marked this function with the comment line `# typedpython: parallel` - # directly above its `def`. Only a marked function can hold a `ForRange.par`. + # SPEC N-15: the user marked this function `@compiled(parallel=True)` (#358). + # Only a marked function can hold a `ForRange.par`. parallel: bool = False + # SPEC N-8 (#153, #332): the user marked this function `@compiled(fastmath=True)` (reassociation + # and FMA contraction enabled for this function only; NOT CPython-identical). + fastmath: bool = False # SPEC N-12b (#268): the producers whose result this pure function keeps inside one statement # (`check(make(d))`), so each such call builds its objects in a region. Set only by `verify`, # which recomputes it and rejects a function whose declared entries differ @@ -883,6 +886,22 @@ class Module: REGION_SCALAR_KINDS = {Type.I64: "i64", Type.F64: "f64", Type.BOOL: "bool"} +def direct_lists(f: object) -> bool: + """#255 (design/typedpython.md 4.3.6): a pure function with three or more list parameters reads + them element by element, without copy-in or critical section. CPython's critical sections lock + one or two objects, and copying three or more lists in at one instant needs them all locked (the + leader decision of 2026-11-13 excludes any lock outside that API). A pure function only reads + its lists, so each `Index` is one `PyList_GetItemRef` in program order, the interpreter's own + atomic read, and an element of the wrong type deopts (legal: nothing has happened yet); `Len` is + the current length. No entry global (a snapshot read at entry could disagree with later list + reads), no buffer parameter, no CopyArray of a list parameter and no parallel loop: the verifier + (`verify/array-direct`) and cgen refuse them. An impure function with more than two list + parameters stays interpreted (#209).""" + params = getattr(f, "params", ()) + lists = sum(isinstance(p, ArrayParam) and p.buffer is None for p in params) + return getattr(f, "pure", False) is True and lists > 2 + + def region_arg_kind(a: object, ref_class) -> object: """The region kind one `New` argument gives its field: "i64" / "f64" / "bool" for a `Box` of that scalar, "null" for the `None` constant, ("ref", class) for a value of a region class (`ref_class(a)` diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index 3f504651c..f056da353 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -154,27 +154,24 @@ class ModuleResult: interface: str = "" kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked" skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured + c_source: Path | None = None # generate_project only: the module's one `.c` (#113) @dataclass class ProjectResult: modules: list[ModuleResult] + runtime_include: Path | None = None # generate_project only: the dir holding tp_runtime.h @property def rebuilt(self) -> list[ModuleResult]: return [m for m in self.modules if m.status == "rebuilt"] -def compile_project( - sources: list[Path], - project_root: Path, - cache_dir: Path, - *, - python_include: Path | None = None, - ext_suffix: str | None = None, - runtime_dir: Path | None = None, - cost_rule: bool = True, -) -> ProjectResult: +def _project(sources: list[Path], project_root: Path, cache_dir: Path, *, cost_rule: bool, + compile_c, compiler_version: str, platform: str, flags: list[str], + c_mode: bool) -> list[ModuleResult]: + """The cached source -> artefact walk shared by compile_project (artefact: the extension) and + generate_project (artefact: the `.c`).""" sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) verified: dict[Path, list[SkippedItem]] = {} @@ -193,27 +190,12 @@ def lower(path: Path) -> ir.Module: def generate(module: ir.Module, path: Path) -> str: return cgen.generate(module, path, _relative(path, root)) if module.functions else "" - def compile_c(c_source: str, name: str, out_dir: Path) -> Path: - if not c_source: - marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) - marker.write_text("nothing compiled\n") - return marker - return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, - python_include=python_include, ext_suffix=ext_suffix) - - version = COMPILER_VERSION - if runtime_dir is not None: - version += ":" + _tree_hash(Path(runtime_dir)) if not cost_rule: - version += ":nocost" - suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") - include = cbuild._python_include(python_include) - flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] - + compiler_version += ":nocost" cache = incremental.BuildCache(cache_dir) report = cache.build_project( sources, project_root, lower=lower, generate=generate, compile_c=compile_c, - compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + compiler_version=compiler_version, platform=platform, flags=flags) side = cache_dir / "skipped" side.mkdir(exist_ok=True) @@ -230,9 +212,78 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: items = [SkippedItem(**i) for i in data["items"]] except (OSError, ValueError, KeyError, TypeError): skipped, items = {}, [] - ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + art = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + ext, c_file = (None, art) if c_mode else (art, None) kind = ("not_marked" if not frontend.is_opted_in(m.path) - else "compiled" if ext is not None else "nothing_compiled") + else "compiled" if art is not None else "nothing_compiled") out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface, - kind, items)) - return ProjectResult(out) + kind, items, c_file)) + return out + + +def _no_c_marker(name: str, out_dir: Path) -> Path: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, + cost_rule: bool = True, +) -> ProjectResult: + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags, + c_mode=False)) + + +# The platform component of a generated-C cache key. The C does not depend on the host, the target, +# the C compiler or its flags (cgen reads none of them; the target's headers and the toolchain are +# the compile slot's inputs and its own cache key, pypackpack docs/design/compile-slot.md §9), so +# one constant stands for all of them and a `.c` built on one machine is reused on another. +GENERATED_C_PLATFORM = "generated-c" + + +def generate_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + cost_rule: bool = True, +) -> ProjectResult: + """Source -> C for every module, cached (#113): the input of pypackpack's native compile slot, + which builds the extensions and caches C -> extension itself. Each compiled module gets exactly + one `.c` (`ModuleResult.c_source`, named after the module's last dotted part); a module with + nothing compiled gets none and ships as Python. `runtime_include` is the directory holding the + header-only `tp_runtime.h`, for the slot's include dirs. The key is the source, the dependency + interfaces and COMPILER_VERSION (which covers the runtime header); no toolchain flag enters it. + An unchanged module is reused with a byte-identical `.c`.""" + def write_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + c_file = Path(out_dir) / (name.rpartition(".")[2] + ".c") + c_file.write_text(c_source) + return c_file + + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=write_c, + compiler_version=COMPILER_VERSION, platform=GENERATED_C_PLATFORM, flags=[], c_mode=True), + runtime_include=cbuild.RUNTIME_DIR) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/resume.py b/python-multiplatform-ksp/src/main/python/typedpython/resume.py index 0a87c5db2..de419c3c3 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/resume.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/resume.py @@ -54,7 +54,11 @@ def __tp_resume_def(src, lmap): def has_snapshot(f: ir.Function) -> bool: """Same rule as `cgen._FnGen.snapshot` (cgen.py, property `snapshot`, ~line 2186): any - entry global, or any ArrayParam (buffer or not; the property does not look at `buffer`).""" + entry global, or any ArrayParam (buffer or not; the property does not look at `buffer`), except + the lists of a function that reads them element by element (`ir.direct_lists`, #255).""" + if ir.direct_lists(f): + return bool(f.entry_globals) or any(isinstance(p, ir.ArrayParam) and p.buffer is not None + for p in f.params) return bool(f.entry_globals) or any(isinstance(p, ir.ArrayParam) for p in f.params) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 9c30cdad8..c6b967a47 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -82,6 +82,14 @@ Python error set, nothing to undo), `-1` Python error set (CPython's exception t typedef struct { int n; /* + a PyCriticalSection2 on free-threaded builds */ } tp_cs; void tp_cs_begin(tp_cs *c, PyObject *a, PyObject *b); void tp_cs_end(tp_cs *c); /* idempotent: a no-op when nothing is held */ + /* #255, ir.direct_lists: a pure function with three or more list parameters does not copy them + in and takes no lock (the exception to "more than two ... not compiled" above). The wrapper + checks PyList_CheckExact and sets only `list` of each tp_*_array; every Index is one of these + (PyList_GetItemRef, the interpreter's read; a negative index counts from the length at that + read): 0 = *out set, -1 = IndexError "list index out of range", 1 = deopt (not exactly float / + not exactly an int within i64). Len is PyList_GET_SIZE at that point. */ + int tp_list_ld_f64(PyObject *list, int64_t i, double *out); + int tp_list_ld_i64(PyObject *list, int64_t i, int64_t *out); `tp_*_array_enter` / `_exit` read and write the list through borrowed references (`PyList_GET_ITEM`, `PyList_SetItem`) and REQUIRE the caller to hold that critical section. Between enter and exit no other thread can change the list, so the whole call is one atomic step as far as lists go (an @@ -114,7 +122,8 @@ call by the export. Rules that go with it (verifier / front end): * Closed (#203) like a list: a function with a buffer runs no user code (`verify/array-open`). -* No limit on the number of buffer parameters (no critical section); lists keep the limit of two. +* No limit on the number of buffer parameters (no critical section); lists keep the limit of two in a + function with an effect (a pure function reads three or more element by element, #255). A buffer and a list cannot alias (different types). Two buffers may share memory (the same array passed twice): per-element access makes reads see earlier writes in program order, as in the interpreter, which a copy-in could not. @@ -261,6 +270,21 @@ per module (the header is `static`) and per thread (`TP_THREAD_LOCAL`): frames o compiled module that sit between two frames of this one are not counted, and another thread's frames never are. + Py_ssize_t tp_seg_depth(PyThreadState *ts, tp_tls *t); /* the segment's Python depth: measured (tp_python_depth) at its first check, then reused */ + void tp_depth_seg_begin(tp_tls *t); /* seg = -1; the wrapper, right before its impl call */ + void tp_depth_seg_end(tp_tls *t); /* seg = -1; the wrapper, right after its impl call */ +Segments (#125, agent proposal, not confirmed): `tp_enter_call` and `tp_depth_room` take the Python +depth from `tp_seg_depth`, so one impl call from its wrapper reads the top frame +(`PyThreadState_GetFrame`) once, not once per compiled frame. Sound because the Python frames below +compiled frames cannot change while they run: Python code started from compiled code (callback, +`__del__`, signal handler, redo) pops every frame it pushed before it returns. The wrapper's two +resets make a nested wrapper measure its own depth and make the outer segment measure again at its +next check after a nested wrapper returned, which is where `tp_python_depth` would have found a +different top frame before; the frame cache therefore changes at exactly the same points. +`Py_GetRecursionLimit()` and `Py_EnterRecursiveCall` stay per frame (a `__del__` can change the limit +between two frames; the C-stack guard compares each frame's stack address). `-DTP_TRACE_DEPTH` +(tests only) adds `__tp_trace_frame_reads__()`: the number of top-frame reads. + ## Fast entry (bounded functions; issue #147) int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k); /* 1 iff tp_python_depth(ts) + count + k <= Py_GetRecursionLimit() */ void tp_trace_fast_hit(void); /* counts a fast entry; a no-op unless -DTP_TRACE_FAST */ @@ -389,7 +413,12 @@ Slot access and the slow paths: int tp_field_set(PyObject *obj, PyMemberDef *m, PyObject *v); /* PyMember_SetOne: 0 / -1 */ int tp_setattr(PyObject *obj, PyObject *name, PyObject *v); /* STORE_ATTR (PyObject_SetAttr): 0 / -1 */ PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); - /* cls->tp_alloc(cls, 0) then set each slot (new references); NULL + MemoryError */ + /* cls->tp_alloc(cls, 0) then a release store of a new reference into each slot + (the fresh object is unreachable until it returns, #125); NULL + MemoryError */ + PyObject *tp_new_fixed_move(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); + /* the same stores with values[i] moved in, not copied: on success the object owns them + and the caller empties its temporaries; on failure nothing was taken (#125). + This is what the generator emits for New */ int tp_is_exact(PyObject *obj, PyTypeObject *cls); /* Py_IS_TYPE */ FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. @@ -482,6 +511,7 @@ static or module-state field must be added here): | `buf_types` (array.array) | module state | immutable after init | written by `tp_exec` (and `tp_clear`); read-only type pointers compared by identity | | buffer arguments of array parameters | caller's array.array memory | atomic | relaxed 8-byte `__atomic_load_n` / `__atomic_store_n` per element; the Py_buffer export is held for the call (open point on concurrent resize, #149) | | list arguments of array parameters | caller's lists | critical section | `tp_cs_begin` before the copy-in, `tp_cs_end` after the write-back, on every path | +| list arguments of a pure function with three or more (#255) | caller's lists | CPython | not copied, no lock: every element read is `PyList_GetItemRef` (atomic, a strong reference), in program order; the wrapper's arguments keep the lists alive | | module globals (class names, entry globals) | module dict | strong reads | `tp_global` and `tp_class_global_ok` use `PyDict_GetItemRef` | | a compiled class's instances | other threads' objects | CPython | `PyMember_GetOne` / `SetOne` are what the interpreter's member descriptors call (they take the object's lock on free-threaded builds); `tp_new_fixed` writes only its own fresh object | | `errno` (math helpers) | C library | thread-local | per thread by POSIX | diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 8b9f54d8e..d28f5b6dd 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1005,10 +1005,10 @@ static inline int tp_unbox_bool(PyObject *o, int *out) * borrowed references (PyList_GET_ITEM) and rely on that lock: no other thread can change the list * between the copy-in and the write-back, so the whole call is one atomic step as far as lists are * concerned (CPython can produce that interleaving: every list operation takes the same lock). - * Liveness note (issue #152): a function holding an array does not poll the eval breaker, so a long - * compiled array loop delays other threads' stop-the-world requests (the free-threaded GC) until it - * returns; results are unaffected. Polls inside such functions need the re-validation decided in - * #152 (after a poll the critical section is re-taken and the list may have changed). + * Polls (issue #152, #335, SPEC N-8): a function holding an array polls once per TP_POLL_INTERVAL + * at the top of a loop iteration: it writes the array back with exit, ends the critical section, + * polls, and enters again (re-taking the section and re-checking the list), so each poll costs a + * full write-back and copy-in of the list (#406). * exit: writes the dirty slots back as new objects (PyList_SetItem steals the new reference and, on * failure, drops it), always frees, and zeroes the struct. On a write-back failure it stops, * frees, and returns -1 with that error set (MemoryError, or IndexError if the list shrank). @@ -1165,6 +1165,56 @@ static inline int tp_i64_array_exit(tp_i64_array *a) return rc; } +/* ------------------------------------------------------------------------------------------ + * Lists read element by element (#255, ir.direct_lists): a pure function with three or more list + * parameters does not copy them in. The wrapper sets only `list` of each tp_*_array (borrowed; the + * call's arguments keep it alive) and every Index is one of these, in program order: the list's own + * PyList_GetItemRef, which is the read the interpreter's `a[i]` does (Objects/listobject.c + * list_get_item_ref; Python/bytecodes.c _BINARY_OP_SUBSCR_LIST_INT), so each read is atomic and no + * lock is held between two reads. A negative index counts from the length at that read, as in the + * interpreter. rc 0 = *out set; -1 = IndexError("list index out of range"); 1 = deopt (the element + * is not exactly float / not exactly an int within i64; nothing has happened: the function is pure). + * Len is PyList_GET_SIZE at that point (an atomic load on free-threaded builds). + * ------------------------------------------------------------------------------------------ */ +static inline PyObject *tp__list_item_ref(PyObject *list, int64_t i) +{ + if (i < 0) { + i += (int64_t)PyList_GET_SIZE(list); + } + if (i < 0 || i > (int64_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GetItemRef(list, (Py_ssize_t)i); /* new reference, or NULL + IndexError */ +} + +static inline int tp_list_ld_f64(PyObject *list, int64_t i, double *out) +{ + PyObject *v = tp__list_item_ref(list, i); + if (v == NULL) { + return -1; + } + if (!PyFloat_CheckExact(v)) { + Py_DECREF(v); + return 1; + } + *out = PyFloat_AS_DOUBLE(v); + Py_DECREF(v); + return 0; +} + +static inline int tp_list_ld_i64(PyObject *list, int64_t i, int64_t *out) +{ + int rc; + PyObject *v = tp__list_item_ref(list, i); + if (v == NULL) { + return -1; + } + rc = tp_unbox_i64(v, out); + Py_DECREF(v); + return rc < 0 ? -1 : rc > 0 ? 1 : 0; +} + /* ------------------------------------------------------------------------------------------ * Buffer arrays (issue #149; ir.ArrayParam.buffer): an exact array.array argument * is NOT copied. tp_buf_enter takes one Py_buffer export and tp_buf_exit releases it; the wrapper @@ -2756,7 +2806,22 @@ static inline int tp_setattr(PyObject *obj, PyObject *name, PyObject *v) /* The observable result of a trivial __init__: allocate with cls's tp_alloc and set slots[i] to * values[i] (new references; the caller keeps its own). NULL with the error set on failure; the - * half-built object is released (its unset slots are NULL, which dealloc handles). */ + * half-built object is released (its unset slots are NULL, which dealloc handles). + * + * Issue #125: each slot is filled by a release store of a new reference, which is what + * PyMember_SetOne does for a Py_T_OBJECT_EX slot (FT_ATOMIC_STORE_PTR_RELEASE of Py_XNewRef(v) + * under the object's critical section, then Py_XDECREF of the old value), minus the critical section + * and the old value. Why that is the same result: + * - every slot was checked at capture (tp_class_capture): Py_T_OBJECT_EX, writable, offset inside + * tp_basicsize; so SetOne's READONLY and type dispatch are decided already; + * - tp_alloc zeroed the object, so the old value is NULL and there is nothing to release; + * - nothing between tp_alloc and the last store can reach a safepoint (no call that detaches the + * thread state, no eval-breaker check), so no stop-the-world (gc.get_objects, gc.get_referrers) + * can hand the object to another thread before it is filled. The critical section, which could + * detach when contended, protected nothing here. A reader that did find it (none can) would see + * NULL or the stored value, as with SetOne's release store. + * A NULL value (the generator never passes one) keeps SetOne's result: AttributeError for deleting + * an unset Py_T_OBJECT_EX slot. */ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n) { @@ -2767,10 +2832,58 @@ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slot return NULL; } for (i = 0; i < n; i++) { - if (PyMember_SetOne((char *)obj, slots[i], values[i]) < 0) { + if (values[i] == NULL) { + PyErr_SetString(PyExc_AttributeError, slots[i]->name); Py_DECREF(obj); return NULL; } + __atomic_store_n((PyObject **)((char *)obj + slots[i]->offset), Py_NewRef(values[i]), + __ATOMIC_RELEASE); + } + return obj; +} + +/* tp_new_fixed with the caller's references moved into the slots (#125, design 4.3.4 "참조 옮기기"): + * on success the object owns values[i] (the caller drops its pointers without releasing them); on + * failure nothing was taken and the caller still owns every values[i]. + * + * Same stores, same object, same allocation as tp_new_fixed; what is gone is a Py_NewRef of each + * value here and the caller's Py_CLEAR of its temporary right after, a +1/-1 pair on the same object + * with nothing in between that can run code or reach a safepoint. Every reference count another + * thread can read during the call is one it could read with the pair (outside the pair's window), so + * no observation tells the two apart; the count after the call is the same. On failure the slots + * already written are emptied again before the half-built object is released, so its dealloc drops + * no reference the caller still owns. + * TP_MOVED_REF is the single mutation point: test_alloc_preserving.py builds with -DTP_MUTANT_COPY_MOVED, + * which defines it as Py_NewRef (a copy instead of a move), leaks one reference per slot and must fail. */ +#ifndef TP_MOVED_REF +#ifdef TP_MUTANT_COPY_MOVED /* the mutation: a copy instead of a move (an object-like switch, as cl.exe + * cannot define a function-like macro on its command line) */ +#define TP_MOVED_REF(v) Py_NewRef(v) +#else +#define TP_MOVED_REF(v) (v) +#endif +#endif +static inline PyObject *tp_new_fixed_move(PyTypeObject *cls, PyMemberDef *const *slots, + PyObject *const *values, Py_ssize_t n) +{ + Py_ssize_t i, j; + PyObject *obj = cls->tp_alloc(cls, 0); + + if (obj == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + if (values[i] == NULL) { + for (j = 0; j < i; j++) { /* give the moved references back to the caller */ + *(PyObject **)((char *)obj + slots[j]->offset) = NULL; + } + PyErr_SetString(PyExc_AttributeError, slots[i]->name); + Py_DECREF(obj); + return NULL; + } + __atomic_store_n((PyObject **)((char *)obj + slots[i]->offset), TP_MOVED_REF(values[i]), + __ATOMIC_RELEASE); } return obj; } @@ -2794,6 +2907,7 @@ typedef struct { Py_ssize_t count; /* compiled frames in flight on this thread (this module) */ PyObject *frame; /* strong ref: the top Python frame when `pydepth` was measured */ Py_ssize_t pydepth; /* Python frames below and including `frame` */ + Py_ssize_t seg; /* Python depth of the current segment, -1 until measured (tp_seg_depth) */ } tp_depth_state; #ifndef TP_POLL_INTERVAL @@ -2859,17 +2973,26 @@ typedef struct { tp_osr *osr; /* the OSR context for the impl about to be entered (#152), or NULL */ } tp_tls; -static TP_THREAD_LOCAL tp_tls tp_tls_v = {{0, NULL, 0}, TP_POLL_INTERVAL, 0, NULL}; +static TP_THREAD_LOCAL tp_tls tp_tls_v = {{0, NULL, 0, -1}, TP_POLL_INTERVAL, 0, NULL}; static inline tp_tls *tp_tls_get(void) { return &tp_tls_v; } +#ifdef TP_TRACE_DEPTH +/* Test builds only (-DTP_TRACE_DEPTH): how many times tp_python_depth read the top frame. */ +static Py_ssize_t tp_trace_frame_reads_count = 0; +#define TP_TRACE_FRAME_READ() (tp_trace_frame_reads_count++) +#else +#define TP_TRACE_FRAME_READ() ((void)0) +#endif + /* Python frame depth of the running thread: public API only. The walk is O(depth), so it is * cached by the identity of the top frame (kept alive while compiled frames are in flight). */ static inline Py_ssize_t tp_python_depth(PyThreadState *ts, tp_tls *t) { + TP_TRACE_FRAME_READ(); PyFrameObject *top = PyThreadState_GetFrame(ts); /* new ref or NULL */ if (top == NULL) { return 0; @@ -2894,9 +3017,35 @@ static inline Py_ssize_t tp_python_depth(PyThreadState *ts, tp_tls *t) return depth; } +/* Segments (#125, design 4.3.3b, agent proposal, not confirmed). A segment is one impl call made + * by its wrapper, up to the return of any wrapper of this module called inside it. Within a + * segment the Python frames below the compiled frames do not change: whatever Python code runs in + * between (a callback, a __del__, a signal handler, a redo) pushes frames and pops every one of them + * before it returns to C, and the frames below cannot return while C above them runs. So the first + * depth check of a segment measures (tp_python_depth: one PyThreadState_GetFrame, and the walk when + * the top frame changed) and every later check reuses `seg`: exactly the value tp_python_depth would + * return there, with the frame cache in exactly the state it would leave. The wrapper resets `seg` + * right before its impl call (a nested wrapper must not take the outer segment's depth) and right + * after it (the outer segment then measures again at its next check, where it measured before + * segments existed: the cached frame, and so the lifetime of a returned callee's frame object, is + * unchanged). Not moved, per frame as before: Py_GetRecursionLimit (a __del__ or a callback can call + * sys.setrecursionlimit between two frames) and Py_EnterRecursiveCall (the C-stack guard compares + * this frame's stack address). Thread-local like `count`; both assume calls nest in LIFO order on a + * thread's C stack. */ +static inline Py_ssize_t tp_seg_depth(PyThreadState *ts, tp_tls *t) +{ + if (t->depth.seg < 0) { + t->depth.seg = tp_python_depth(ts, t); + } + return t->depth.seg; +} + +static inline void tp_depth_seg_begin(tp_tls *t) { t->depth.seg = -1; } +static inline void tp_depth_seg_end(tp_tls *t) { t->depth.seg = -1; } + static inline int tp_enter_call(PyThreadState *ts, tp_tls *t) { - if (tp_python_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) { + if (tp_seg_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); return -1; } @@ -2935,7 +3084,7 @@ static inline void tp_leave_call(tp_tls *t) * compiled calls started from it see the true depth. */ static inline int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k) { - return tp_python_depth(ts, t) + t->depth.count + k <= Py_GetRecursionLimit(); + return tp_seg_depth(ts, t) + t->depth.count + k <= Py_GetRecursionLimit(); } /* After a fast tree: tp_python_depth may have cached the top frame (a strong reference) and, with @@ -2974,6 +3123,16 @@ static PyObject *tp_trace_fast_hits(PyObject *tp_module, PyObject *tp_unused) static inline void tp_trace_fast_hit(void) {} #endif +#ifdef TP_TRACE_DEPTH +/* Test builds only: `__tp_trace_frame_reads__()`, installed by the exec slot. */ +static PyObject *tp_trace_frame_reads(PyObject *tp_module, PyObject *tp_unused) +{ + (void)tp_module; (void)tp_unused; + return PyLong_FromSsize_t(tp_trace_frame_reads_count); +} +static PyMethodDef tp_trace_frame_reads_def = {"__tp_trace_frame_reads__", tp_trace_frame_reads, METH_NOARGS, NULL}; +#endif + /* ------------------------------------------------------------------------------------------ * Eval breaker (issue #141). The interpreter checks between bytecodes for signals, pending calls, * GIL drop requests, async exceptions and scheduled collections; compiled code has no bytecodes. @@ -2984,11 +3143,11 @@ static inline void tp_trace_fast_hit(void) {} * (forced GIL switching to a waiting thread, KeyboardInterrupt, gc). Public API only. * * Signal handlers, pending calls and other threads are user code that may change lists and - * globals. A function holding an array copy-in or an entry-globals snapshot runs no user code by - * contract (ir.ArrayParam, ir.Function.entry_globals), so such a function does not poll, and while - * one is active (`tp_snapshot_depth > 0`, counted at its entry and exit) the slow path does nothing: - * the event stays pending and is handled at the next poll or bytecode after it returns, which is the - * CPython execution in which it arrived late. + * globals. A function holding an array copy-in or an entry-globals snapshot runs no user code while + * it holds them (ir.ArrayParam, ir.Function.entry_globals): while one is active + * (`tp_snapshot_depth > 0`, counted at its entry and exit) the slow path does nothing and the event + * stays pending. Its own polls (#152, #335) first write the arrays back and lower the depth, so the + * handler runs there with the lists up to date, and the function copies them in again after. * ------------------------------------------------------------------------------------------ */ #ifndef TP_POLL_INTERVAL #define TP_POLL_INTERVAL 4096 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index 6493c8f93..b54f31247 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -37,14 +37,17 @@ verify/structure break/continue outside a loop, the ForRange var assigned in its body, a zero or non-constant step, duplicate names, bad declarations, an entry global that collides with a parameter/local/other entry global or - is assigned in the body, or more than two array parameters (the lists - are locked from copy-in to write-back: one- and two-object critical - sections only, #159) + is assigned in the body, or more than two array parameters in a + function that is not pure (the lists are locked from copy-in to + write-back: one- and two-object critical sections only, #159) verify/proven-overflow `proven` is set on an op the verifier cannot prove stays inside i64 (or on a node `proven` does not apply to, or it is not a bool) verify/redo `Call.redo` where it is not legal (see below) verify/entry-global-open a function with entry_globals is not closed verify/array-open a function with an ArrayParam runs user code (#203) + verify/array-direct a pure function with three or more list parameters (#255, + `ir.direct_lists`: read element by element) has an entry global, a + buffer parameter, a CopyArray of a list parameter or a parallel loop verify/buffer a malformed `ArrayParam.buffer` (None, "array", "numpy", "memoryview"), or a CopyArray of a buffer array (#149) verify/par-loop a `ForRange.par` (SPEC N-15, #277) that the verifier does not recompute to @@ -308,6 +311,7 @@ REDO = "verify/redo" ENTRY_OPEN = "verify/entry-global-open" ARRAY_OPEN = "verify/array-open" +ARRAY_DIRECT = "verify/array-direct" BUFFER = "verify/buffer" CLASS = "verify/class" FIELD = "verify/field-unproved" @@ -402,6 +406,7 @@ def __init__(self, function: Function, signatures: dict[str, Function], classes: dict[str, ClassDecl] | None = None, var_ivs: dict[str, tuple[int, int]] | None = None, fixing: bool = False): self.fn = function + self.direct = ir.direct_lists(function) # #255: lists read element by element # Local intervals (module docstring "Local variables and guards"): `var_ivs` is the interval # of every eligible local; `fixing` says this walk is a round of the fixpoint that computes # them (a local without an interval yet is then "bottom": `bottom` is set when one is read). @@ -469,7 +474,15 @@ def declarations(self) -> frozenset[str]: if p.name in params: self.error(STRUCTURE, f"parameter '{p.name}' appears twice") params.add(p.name) - if sum(type(p) is ArrayParam and p.buffer is None for p in fn.params) > 2: + if self.direct: + self.can_deopt = True # every element read checks the element's type (#255) + if isinstance(fn.entry_globals, tuple) and fn.entry_globals: + self.error(ARRAY_DIRECT, "three or more list parameters are read element by element " + "(#255), so an entry global cannot be read once at entry") + if any(type(p) is ArrayParam and p.buffer is not None for p in fn.params): + self.error(ARRAY_DIRECT, "three or more list parameters are read element by element " + "(#255); a buffer parameter beside them is not supported") + elif sum(type(p) is ArrayParam and p.buffer is None for p in fn.params) > 2: self.error(STRUCTURE, "more than two array parameters: compiled code holds each list's " "critical section from copy-in to write-back, and free-threaded " "CPython has no section for more than two objects (#159)") @@ -1194,6 +1207,11 @@ def index(self, array: str, index: object, state: _State, node: str def e_Index(self, e: Index, state: _State) -> ir.Expr: idx, element, proven = self.index(e.array, e.index, state, "index") + if self.direct and e.array in self.arrays and e.array not in self.buffers: + # #255: another thread can resize the list between two reads, so no bound is proved, + # and an element of another type deopts + proven = False + self.can_deopt = True if element is not None and e.type is not element and e.type is not Type.OBJ: self.error(TYPE, f"element of '{e.array}' is {element.value}, node says " f"{e.type.value}") @@ -1372,6 +1390,9 @@ def allocation(self, e: NewArray | CopyArray, target: str, state: _State) -> ir. elif e.src in self.buffers: self.error(BUFFER, f"CopyArray of buffer array '{e.src}': a buffer array is never " "copied)") + elif self.direct and e.src in self.arrays: + self.error(ARRAY_DIRECT, f"CopyArray of list parameter '{e.src}' in a function that reads " + "its three or more lists element by element (#255)") else: self.array(e.src, "copy", state) if src is not e.type: @@ -1673,7 +1694,11 @@ def s_ForRange(self, s: ForRange, state: _State): for br in loop.breaks: after = _meet(after, br) par = self._par_info(s, step_ok, body, loop, state) - if s.par is not None and s.par != par: + if self.direct and (par is not None or s.par is not None): + self.error(ARRAY_DIRECT, "a parallel loop in a function that reads its three or more lists " + "element by element (#255)") + par = None + elif s.par is not None and s.par != par: self.error(PAR_LOOP, f"ForRange over '{var}' declares par={s.par!r}, but the verifier " f"computes {par!r} (SPEC N-15)") return dataclasses.replace(s, start=start, stop=stop, body=body, par=par), after diff --git a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth index 98776970a..1b8b0a0ad 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth +++ b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth @@ -1 +1 @@ -import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 452a4e613..975fdc08b 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -1,12 +1,36 @@ +import ast +import builtins import sys from pathlib import Path import textwrap import pytest +# The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. +builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) + +# Prepended to every child-process script that loads a module whose source uses `@compiled`: a plain +# CPython has the builtin only when `typedpython_builtins.pth` is installed, so the child installs it. +COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f=None, /, **_k: (lambda g: g) if f is None else f\n" + FIXTURES = Path(__file__).parent / "fixtures" +def decorate_compiled(source: str) -> str: + """`source` with `@compiled` put above every module-level def / class (above its other decorators), + the set a whole-module mark used to cover. A source that does not parse is returned unchanged.""" + try: + tree = ast.parse(source) + except SyntaxError: + return source + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + @pytest.fixture def write(tmp_path): """Write a dedented source file under tmp_path and return its path.""" diff --git a/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py new file mode 100644 index 000000000..2b43b4cdd --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py @@ -0,0 +1,476 @@ +"""TypedPython object cost without a change in what Python can observe (#125, design 4.3.4 +"참조 옮기기와 빌린 피연산자", cgen module docstring rules 3 and 3a). + +Two changes of the generated C, both removing a +1/-1 reference-count pair on an object that another +reference keeps alive, with nothing in between that can run code or reach a safepoint: + - a New moves its argument temporaries' references into the fresh object's slots + (`tp_new_fixed_move`) instead of copying them and releasing the temporaries; + - an operand a helper only borrows (the two sides of Is, the object of FieldGet / IsExact, an OBJ + argument of a C to C Call) names the OBJ local itself instead of a Py_NewRef copy. + +The behaviour tests compare every observation the leader's constraint (2026-11-13) lists, gc +tracking, gc.get_objects, weakrefs and their callbacks, __del__ order, sys.getrefcount, id +uniqueness, subclasses and a class changed after import, with the interpreted functions the +extension keeps. They pass before and after the change: they guard the preserved behaviour, they do +not measure the change. The mutation tests show that the guarded properties fail when the move or the +borrow is done wrongly (a copy that leaks, a release that frees too early). + +Red record (expected; agents do not build here, AGENTS.md 10, so CI runs this file): + - before the change: test_gen_new_moves_*, test_gen_borrowed_operands_*, test_header_move_* + fail (no tp_new_fixed_move, Py_NewRef copies of locals), and the three mutation tests fail with + AttributeError (no cgen._moved_temps / cgen._operand_release) or a compile error (no + TP_MOVED_REF); every behaviour test passes; + - after the change: everything passes. +""" +from __future__ import annotations + +import gc +import subprocess +import sys +import textwrap +import threading +import weakref + +import pytest + +from test_cgen import PACKAGE, OUT_ROOT, BOOL, OBJ, L, fn, runtime_dir, need_runtime +from test_cgen_classes import f_make, f_check +from typedpython.ir import ClassDecl, FieldGet, Is, Module, New, Param, Return, Const + +SOURCE = textwrap.dedent('''\ + """typedpython allocation test module, the interpreted reference.""" + LOG = [] + + class Node: + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + class WNode: # weak-referenceable + __slots__ = ("left", "right", "__weakref__") + + def __init__(self, left, right): + self.left = left + self.right = right + + class DNode: # a finalizer from the start + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + def __del__(self): + LOG.append(id(self)) + + class LNode: # gains __del__ after import (a test sets it) + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + def make(depth): + if depth == 0: + return None + return Node(make(depth - 1), make(depth - 1)) + + def check(node): + if node is None: + return 0 + return 1 + check(node.left) + check(node.right) + + def make_w(depth): + if depth == 0: + return None + return WNode(make_w(depth - 1), make_w(depth - 1)) + + def make_d(depth): + if depth == 0: + return None + return DNode(make_d(depth - 1), make_d(depth - 1)) + + def make_l(depth): + if depth == 0: + return None + return LNode(make_l(depth - 1), make_l(depth - 1)) + + def pair(x): + return Node(x, x) + + def first(p): + return p.left + + def is_none(x): + return x is None + ''') + +CLASSES = (ClassDecl("Node", ("left", "right"), True), ClassDecl("WNode", ("left", "right"), True), + ClassDecl("DNode", ("left", "right"), True, has_del=True), + ClassDecl("LNode", ("left", "right"), True)) + + +def f_pair(): + x = L("x", OBJ) + return fn("pair", [Param("x", OBJ)], OBJ, {}, [Return(New(OBJ, "Node", (x, x)))], + pure=True, may_deopt=True) + + +def f_first(): + return fn("first", [Param("p", OBJ, cls="Node")], OBJ, {}, + [Return(FieldGet(OBJ, L("p", OBJ), "Node", "left"))], pure=True, may_deopt=True) + + +def f_is_none(): + return fn("is_none", [Param("x", OBJ)], BOOL, {}, + [Return(Is(BOOL, L("x", OBJ), Const(OBJ, None)))], pure=True, may_deopt=False) + + +def functions(): + return (f_make(), f_check(), f_make("make_w", "WNode"), f_make("make_d", "DNode"), + f_make("make_l", "LNode"), f_pair(), f_first(), f_is_none()) + + +def generate(name): + import shutil + from typedpython import cgen + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return cgen.generate(Module(name=name, functions=functions(), classes=CLASSES), p) + + +def build(name, flags=None): + from typedpython import cbuild + need_runtime() + src = generate(name) + kwargs = {} if flags is None else {"flags": flags} + so = cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir(), **kwargs) + return cbuild.load(name, so), so + + +@pytest.fixture(scope="module") +def m(): + return build("tp_alloc_pres")[0] + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def nodes(t): + """Every node of a tree, pre-order.""" + out, todo = [], [t] + while todo: + n = todo.pop() + if n is None: + continue + out.append(n) + todo.append(n.right) + todo.append(n.left) + return out + + +def impl_body(text, name): + """The definition (not the forward declaration) of the counted impl of `name`.""" + sig = f"static int tp_impl_{name}(PyObject *tp_module, PyThreadState" + start = text.index(sig) + while text[start:text.index("\n", start)].endswith(";"): # skip the declaration + start = text.index(sig, start + 1) + return text[start:text.index("\n}\n", start)] + + +# --- the generated C (red before the change) -------------------------------------------------------- + +def test_gen_new_moves_its_argument_temporaries(): + text = generate("tp_alloc_gen_new") + assert "tp_new_fixed(" not in text, "a New still copies its arguments (tp_new_fixed)" + lines = text.splitlines() + sites = [i for i, ln in enumerate(lines) if "= tp_new_fixed_move(" in ln] + assert sites, text + for i in sites: + t = lines[i].split("=")[0].strip() + nxt = lines[i + 1].strip() + assert nxt.startswith(f"if ({t} != NULL) {{") and "= NULL;" in nxt, (lines[i], nxt) + + +def test_gen_borrowed_operands_take_no_copy(): + text = generate("tp_alloc_gen_borrow") + check = impl_body(text, "check") + # adoption of the parameter stays (rule 1); no copy of the local + assert "l_node = Py_NewRef(p_node);" in check and "Py_NewRef(l_node)" not in check, check + assert "(l_node == Py_None)" in check, check + assert "tp_field_get(l_node," in check, check + assert "tp_impl_check(tp_module, tp_ts, tp_t, l_l," in check, check + assert "tp_impl_check(tp_module, tp_ts, tp_t, l_r," in check, check + assert "Py_NewRef(Py_None)" not in check, check + assert "tp_release(&l_" not in check.split("tp_exit:")[0], "a borrowed local is released" + first = impl_body(text, "first") + assert "tp_field_get(l_p," in first and "Py_NewRef(l_p)" not in first, first + + +def test_header_move_helper_moves_and_gives_back_on_failure(): + from typedpython import cbuild + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + start = h.index("static inline PyObject *tp_new_fixed_move(") + body = h[start:h.index("\n}\n", start)] + assert "Py_NewRef(" not in body, body # the caller's reference moves in + assert "TP_MOVED_REF(values[i])" in body, body # the single mutation point + assert "__atomic_store_n(" in body and "__ATOMIC_RELEASE" in body, body + assert "PyMember_SetOne" not in body and "CRITICAL_SECTION" not in body, body + # on a failure after some stores, those slots are emptied before the object is released + fail = body[body.index("values[i] == NULL"):] + assert fail.index("= NULL;") < fail.index("Py_DECREF(obj)"), body + + +# --- observable behaviour, compiled against interpreted (green before and after) ------------------- + +def test_results_match_the_interpreter(m): + for d in (0, 1, 5): + assert m.check(m.make(d)) == interp(m, "check")(interp(m, "make")(d)) == 2 ** d - 1 + x = object() + p, q = m.pair(x), interp(m, "pair")(x) + assert type(p) is type(q) is m.Node and p.left is p.right is x and q.left is x + assert m.first(p) is x and m.is_none(None) is True and m.is_none(p) is False + + +def test_compiled_objects_are_gc_tracked_like_interpreted_ones(m): + t, u = m.make(4), interp(m, "make")(4) + assert [gc.is_tracked(n) for n in nodes(t)] == [gc.is_tracked(n) for n in nodes(u)] + assert all(gc.is_tracked(n) for n in nodes(t)) + assert gc.is_tracked(m.pair(None)) and gc.is_tracked(interp(m, "pair")(None)) + + +def test_compiled_objects_are_in_gc_get_objects(m): + t = m.make(5) + live = {id(o) for o in gc.get_objects() if type(o) is m.Node} + assert all(id(n) in live for n in nodes(t)) + assert len(nodes(t)) == 31 + + +def test_ids_are_unique_while_alive(m): + t = m.make(7) + keep = [m.pair(None) for _ in range(100)] + ids = [id(n) for n in nodes(t)] + [id(p) for p in keep] + assert len(ids) == len(set(ids)) == 127 + 100 + + +def test_weakref_callbacks_run_in_the_interpreters_order(m): + def order(make): + t = make(4) + index = {id(n): k for k, n in enumerate(nodes(t))} + log = [] + refs = [weakref.ref(n, lambda r, k=index[id(n)]: log.append(k)) for n in nodes(t)] + assert all(r() is not None for r in refs) + del t + assert all(r() is None for r in refs) + return log + got, want = order(m.make_w), order(interp(m, "make_w")) + assert got == want and sorted(got) == list(range(15)), (got, want) + + +def test_del_runs_once_per_object_in_the_interpreters_order(m): + def order(make): + m.LOG.clear() + t = make(4) + index = {id(n): k for k, n in enumerate(nodes(t))} + assert m.LOG == [] + del t + return [index[i] for i in m.LOG] + got, want = order(m.make_d), order(interp(m, "make_d")) + assert got == want and sorted(got) == list(range(15)), (got, want) + + +def test_a_class_that_gains_del_later_finalizes_compiled_objects(m): + t = m.make_l(3) + m.LNode.__del__ = lambda self: m.LOG.append(id(self)) + try: + m.LOG.clear() + ids = {id(n) for n in nodes(t)} + del t + assert set(m.LOG) == ids and len(m.LOG) == 7 + m.LOG.clear() + u = m.make_l(3) # built after the change (the class's tag moved) + ids = {id(n) for n in nodes(u)} + del u + assert set(m.LOG) == ids and len(m.LOG) == 7 + finally: + del m.LNode.__del__ + m.LOG.clear() + + +def test_a_subclass_with_del_inside_a_compiled_pair_dies_when_the_pair_dies(m): + log = [] + + class Sub(m.Node): + __slots__ = () + + def __del__(self): + log.append("sub") + + s = Sub(None, None) + p = m.pair(s) + del s + assert log == [] # the pair's two slots keep it alive + assert m.check(Sub(None, None)) == interp(m, "check")(Sub(None, None)) == 1 + log.clear() + del p + assert log == ["sub"] + + +def test_reference_counts_match_the_interpreter_at_every_call_boundary(m): + x = object() + base = sys.getrefcount(x) + p = m.pair(x) + compiled = sys.getrefcount(x) + q = interp(m, "pair")(x) + assert compiled == base + 2 and sys.getrefcount(x) == base + 4 + del p, q + n, h = m.Node(None, None), m.Node(x, None) + b = sys.getrefcount(n), sys.getrefcount(h), sys.getrefcount(x) + for _ in range(50): + m.check(n), m.first(h), m.is_none(n), m.check(m.Node(n, n)) + assert (sys.getrefcount(n), sys.getrefcount(h), sys.getrefcount(x)) == b + + +def test_reference_counts_are_unchanged_over_many_calls(m): + shared = object() + holder = m.Node(shared, None) + keep = m.Node(None, None) + + def churn(k): + for _ in range(k): + m.check(m.make(4)) + m.pair(shared) + m.first(holder) + m.is_none(keep) + m.check(m.Node(keep, None)) + + churn(20) + base = sys.getrefcount(shared), sys.getrefcount(holder), sys.getrefcount(keep) + churn(3000) + assert (sys.getrefcount(shared), sys.getrefcount(holder), sys.getrefcount(keep)) == base + + +def test_concurrent_threads_keep_counts_exact(m): + """3.15t: other threads move a shared object's count through ob_ref_shared, this one through + ob_ref_local; a lost or extra increment shows in sys.getrefcount once the threads are done.""" + shared = object() + keep = m.Node(None, None) + base = sys.getrefcount(shared), sys.getrefcount(keep) + errors = [] + + def work(): + try: + for _ in range(3000): + p = m.pair(shared) + assert m.first(p) is shared + assert m.check(m.Node(keep, keep)) == 3 + assert m.check(m.make(3)) == 7 + assert m.is_none(keep) is False + except Exception as e: # noqa: BLE001 (reported below) + errors.append(repr(e)) + + threads = [threading.Thread(target=work) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + assert (sys.getrefcount(shared), sys.getrefcount(keep)) == base + + +# --- mutations: a wrong move or borrow must fail ---------------------------------------------------- + +MUTANT_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + log = [] + class T: + def __del__(self): + log.append("T") + bad = 0 + x = T() + base = sys.getrefcount(x) + p = m.pair(x) + if sys.getrefcount(x) != base + 2: + bad |= 1 + for _ in range(200): + m.pair(x) + m.check(m.make(4)) + if sys.getrefcount(x) != base + 2: + bad |= 2 + del x + if log: + bad |= 4 # freed while the pair still holds it + del p + if log != ["T"]: + bad |= 8 + n = m.Node(None, None) + b = sys.getrefcount(n) + for _ in range(100): + m.check(n) + m.first(n) + m.is_none(n) + if sys.getrefcount(n) != b: + bad |= 16 + print("RESULT", bad, log) + sys.exit(bad) + ''') + + +def run_child(name, so): + script = MUTANT_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=120) + + +def test_mutation_unmutated_build_passes_the_script(): + _, so = build("tp_alloc_ok") + run = run_child("tp_alloc_ok", so) + assert run.returncode == 0, run.stdout + run.stderr + + +def _build_mutant(name, monkeypatch, attr, value): + from typedpython import cgen, cbuild + need_runtime() + monkeypatch.setattr(cgen, attr, value) + src = generate(name) + monkeypatch.undo() + return cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir()) + + +def test_mutation_releasing_moved_temporaries_is_caught(monkeypatch): + """The temporaries a New moved into its object are released anyway: one reference too few per + slot, the argument dies while the pair still points at it.""" + so = _build_mutant("tp_alloc_mut_move", monkeypatch, "_moved_temps", lambda items: "") + run = run_child("tp_alloc_mut_move", so) + assert run.returncode != 0, "the over-release mutant passed:\n" + run.stdout + run.stderr + + +def test_mutation_copying_instead_of_moving_is_caught(): + """tp_new_fixed_move takes a new reference as tp_new_fixed did, while the caller still drops its + temporaries without releasing them: one leaked reference per slot.""" + from typedpython import cbuild + flags = list(cbuild.DEFAULT_FLAGS) + ["-DTP_MUTANT_COPY_MOVED"] # object-like: cl.exe too + need_runtime() + src = generate("tp_alloc_mut_copy") + so = cbuild.build(src, "tp_alloc_mut_copy", OUT_ROOT / "tp_alloc_mut_copy", + runtime_dir=runtime_dir(), flags=flags) + run = run_child("tp_alloc_mut_copy", so) + assert run.returncode != 0, "the leaking mutant passed:\n" + run.stdout + run.stderr + + +def test_mutation_releasing_a_borrowed_local_is_caught(monkeypatch): + """A borrowed OBJ local is released after the helper as if it were a temporary: the local's own + reference is dropped (and the local emptied) while the function still uses it.""" + so = _build_mutant("tp_alloc_mut_borrow", monkeypatch, "_operand_release", + lambda name, owned: name if (owned or name.startswith("l_")) else None) + run = run_child("tp_alloc_mut_borrow", so) + assert run.returncode != 0, "the borrowed-release mutant passed:\n" + run.stdout + run.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py b/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py new file mode 100644 index 000000000..b433d1554 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py @@ -0,0 +1,91 @@ +"""TypedPython: the five benchmarks print exactly what the interpreter prints (#41, M2 criterion). + +The M2 criterion of #41 asks for a record that the compiled benchmarks give output identical to +CPython. `benchmarks/typedpython/run.py verify --typedpython` checks that too, but it also needs +Node, Java and Rust, so it does not run in the TypedPython CI. This file runs the same comparison +inside that CI, on every push to develop, so the record carries the commit and date of each run. + +For each benchmark: the source with `@compiled` above every top-level def and class is compiled +through `pipeline.compile_module`; its `main()` runs at the size `run.py verify` uses, and its stdout +is compared with the plain source run by the interpreter in the same process. The compiled-function +list is printed (`pytest -s` shows it) so a run also records what was compiled; the list itself is +pinned by `test_class_methods.py::test_the_benchmarks_compile_the_same_functions_as_before`. + +What this cannot find: a difference that only shows at the benchmark size, or only under concurrent +callers. The differential tests listed in docs/design/typedpython.md cover the arithmetic edge cases. + +Written after the implementation (a record of existing behaviour, not a red/green test): it passes on +develop today, and a mismatch is a regression. +""" +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import pipeline + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +BENCH = REPO / "benchmarks" / "typedpython" / "py" + +# name -> (source file, size), the verify sizes of benchmarks/typedpython/run.py BENCHMARKS +SIZES = { + "nbody": ("nbody.py", "5000"), + "spectral-norm": ("spectral_norm.py", "100"), + "fannkuch-redux": ("fannkuch.py", "7"), + "binary-trees": ("binary_trees.py", "10"), + "wordfreq": ("wordfreq.py", "100000"), +} + + +def run_main(module, name: str, size: str) -> str: + old = sys.argv + sys.argv = [name, size] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + module.main() + finally: + sys.argv = old + return buf.getvalue() + + +def load_plain(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("name", sorted(SIZES)) +def test_compiled_benchmark_output_equals_the_interpreter(tmp_path, name): + py, size = SIZES[name] + source = (BENCH / py).read_text() + stem = py.removesuffix(".py") + + plain_path = tmp_path / f"{stem}_ref.py" + plain_path.write_text(source) + expected = run_main(load_plain(plain_path, f"{stem}_ref"), name, size) + assert expected.strip(), f"{name}: the interpreted run printed nothing" + + path = tmp_path / py + path.write_text(decorate_compiled(source)) + result = pipeline.compile_module(path, tmp_path / "out") + compiled = sorted(f.name for f in result.module.functions) + print(f"\n{name}: compiled {compiled}, skipped {sorted(result.skipped)}") + if result.extension is None: + # nothing compiled (nbody on free-threaded builds until #255): the comparison would be + # interpreter against interpreter, so it is reported, not counted as a pass + assert compiled == [] + pytest.skip(f"{name}: no function compiled on this build, nothing to compare") + + module = pipeline.load(result) + before = getattr(module, "__typedpython_deopts__", None) + assert run_main(module, name, size) == expected + print(f"{name}: output identical, deopts {before} -> {getattr(module, '__typedpython_deopts__', None)}") diff --git a/python-multiplatform-ksp/src/test/python/test_bitwise.py b/python-multiplatform-ksp/src/test/python/test_bitwise.py index 0bd844034..c0d48b3d4 100644 --- a/python-multiplatform-ksp/src/test/python/test_bitwise.py +++ b/python-multiplatform-ksp/src/test/python/test_bitwise.py @@ -20,33 +20,39 @@ from test_native_int_impure import build_source SOURCE = textwrap.dedent(''' - # typedpython: compiled + @compiled def p_and(a: int, b: int) -> int: return a & b + @compiled def p_or(a: int, b: int) -> int: return a | b + @compiled def p_xor(a: int, b: int) -> int: return a ^ b + @compiled def p_inv(a: int) -> int: return ~a + @compiled def p_shl(a: int, b: int) -> int: return a << b + @compiled def p_shr(a: int, b: int) -> int: return a >> b + @compiled def p_aug(a: int, b: int) -> int: x: int = a x ^= b @@ -55,22 +61,26 @@ def p_aug(a: int, b: int) -> int: return x + @compiled def p_augshl(a: int, b: int) -> int: x: int = a x <<= b return x + @compiled def p_augshr(a: int, b: int) -> int: x: int = a x >>= b return x + @compiled def p_mix(a: int, b: int) -> int: return (((a & 255) << 3) | (b >> 2)) ^ ~a + @compiled def p_mask_loop(n: int) -> int: h: int = 0 for i in range(n): @@ -78,6 +88,7 @@ def p_mask_loop(n: int) -> int: return h + @compiled def p_array(xs: list[int]) -> None: for i in range(len(xs)): xs[i] &= 0xFF @@ -85,10 +96,12 @@ def p_array(xs: list[int]) -> None: xs[i] ^= i + @compiled def p_iv(a: int, b: int) -> int: return (a & 0xFFFF) * (b & 0xFFFF) + ~(a & 7) + @compiled def i_and(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a & b @@ -96,6 +109,7 @@ def i_and(log: list[object], a: int, b: int) -> object: return r + @compiled def i_or(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a | b @@ -103,6 +117,7 @@ def i_or(log: list[object], a: int, b: int) -> object: return r + @compiled def i_xor(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a ^ b @@ -110,6 +125,7 @@ def i_xor(log: list[object], a: int, b: int) -> object: return r + @compiled def i_inv(log: list[object], a: int) -> object: log.append("start") r: int = ~a @@ -117,6 +133,7 @@ def i_inv(log: list[object], a: int) -> object: return r + @compiled def i_shl(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -124,6 +141,7 @@ def i_shl(log: list[object], a: int, b: int) -> object: return r + @compiled def i_shr(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a >> b @@ -131,6 +149,7 @@ def i_shr(log: list[object], a: int, b: int) -> object: return r + @compiled def i_aug(log: list[object], a: int, b: int) -> object: log.append("start") x: int = a @@ -143,6 +162,7 @@ def i_aug(log: list[object], a: int, b: int) -> object: return x + @compiled def i_big(log: list[object], a: int, b: int, c: int) -> object: log.append("start") r: int = (a * b) & c @@ -156,6 +176,7 @@ def i_big(log: list[object], a: int, b: int, c: int) -> object: return r + @compiled def o_ops(log: list[object], x: object, y: object) -> object: log.append("start") r: object = x | y @@ -164,21 +185,25 @@ def o_ops(log: list[object], x: object, y: object) -> object: return r + @compiled def o_inplace(log: list[object], x: object, y: object) -> object: log.append("start") x |= y return x + @compiled def f_and(log: list[object], a: float, b: int) -> object: log.append("start") return a & b + @compiled def b_and(a: bool, b: bool) -> bool: return a & b + @compiled def b_inv(a: bool) -> int: return ~a ''') diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py index 71d95f0c0..2bab5516f 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py @@ -29,10 +29,10 @@ from typedpython import cbuild, cgen, frontend, ir, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -40,6 +40,7 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def sum_aq(a: array.array[int]) -> int: s = 0 for i in range(len(a)): @@ -47,70 +48,85 @@ def sum_aq(a: array.array[int]) -> int: return s + @compiled def get_ad(a: array.array[float], i: int) -> float: return a[i] + @compiled def get_aq(a: array.array[int], i: int) -> int: return a[i] + @compiled def set_ad(a: array.array[float], i: int, v: float) -> None: a[i] = v + @compiled def set_aq(a: array.array[int], i: int, v: int) -> None: a[i] = v + @compiled def length_ad(a: array.array[float]) -> int: return len(a) + @compiled def scale_ad(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def iota_aq(a: array.array[int]) -> None: for i in range(len(a)): a[i] = i + @compiled def copy_ad(dst: array.array[float], src: array.array[float]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def cascade_ad(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + 1.0 + @compiled def fill_n(a: array.array[float], n: int) -> None: for i in range(n): a[i] = 1.0 + @compiled def mixed(l: list[float], a: array.array[float]) -> None: for i in range(len(a)): a[i] = l[i] + @compiled def four(a: array.array[float], b: array.array[float], c: array.array[float], d: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + c[i] + d[i] + @compiled def div_ad(a: array.array[float]) -> float: return a[0] / a[1] + @compiled def open_ad(a: array.array[float], o: object) -> float: return a[0] + float(o.v) + @compiled def rebind_view(a: array.array[float]) -> None: y = a[:] y[0] = 5.0 @@ -493,42 +509,49 @@ def test_verifier_buffer_params_are_not_limited_but_lists_still_are(): from test_verify import F64 as VF64, F64A as VF64A, assert_accepted, assert_rejected, fn from typedpython.ir import ArrayParam, Const, Index, Return, Type - def many(n, buffer): + def many(n, buffer, pure=True, may_deopt=False): return fn("f", params=[ArrayParam(f"a{k}", VF64A, False, buffer) for k in range(n)], - returns=VF64, pure=True, + returns=VF64, pure=pure, may_deopt=may_deopt, body=[Return(Index(VF64, "a0", Const(Type.I64, 0)))]) assert_accepted(many(5, "array")) - assert_rejected("verify/structure", many(3, None)) + # #255 (d1): a pure function reads 3+ lists element by element (it may deopt on an element of + # another type); a function with effects keeps the limit of two lists + assert_accepted(many(3, None, may_deopt=True)) + assert_rejected("verify/structure", many(3, None, pure=False)) # --- vectorised element-wise store loops (#226) --------------------------------------------------- VEC_SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def vscale(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def vmix(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = -(a[i] - k) * 2.5 + k + @compiled def vtwo(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = a[i] + b[i] + @compiled def vfill(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = k + @compiled def vdiv(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] / k diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index 1766b3300..c0862bed1 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -23,10 +23,10 @@ same_value) SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_mf(m: memoryview[float]) -> float: s = 0.0 for i in range(len(m)): @@ -34,6 +34,7 @@ def sum_mf(m: memoryview[float]) -> float: return s + @compiled def sum_mi(m: memoryview[int]) -> int: s = 0 for i in range(len(m)): @@ -41,31 +42,38 @@ def sum_mi(m: memoryview[int]) -> int: return s + @compiled def get_mf(m: memoryview[float], i: int) -> float: return m[i] + @compiled def get_mi(m: memoryview[int], i: int) -> int: return m[i] + @compiled def set_mf(m: memoryview[float], i: int, v: float) -> None: m[i] = v + @compiled def set_mi(m: memoryview[int], i: int, v: int) -> None: m[i] = v + @compiled def length_mf(m: memoryview[float]) -> int: return len(m) + @compiled def scale_mf(m: memoryview[float], k: float) -> None: for i in range(len(m)): m[i] = m[i] * k + @compiled def spin_mf(m: memoryview[float], n: int) -> float: s = 0.0 L = len(m) @@ -74,6 +82,7 @@ def spin_mf(m: memoryview[float], n: int) -> float: return s + @compiled def spin_af(a: array.array[float], n: int) -> float: s = 0.0 L = len(a) @@ -471,11 +480,13 @@ def calibrated_spin(built): def probe_during_call(call, probes, attempts=3): """Run call() on the main thread while a worker applies each probe in a loop. Returns - (result, {probe_name: (BufferError count, other outcomes)}). Retries `attempts` times when a - probe never observed the call (machine too fast or too loaded).""" + (result, {probe_name: (BufferError count, other outcomes, late successes)}). Retries `attempts` + times when a probe never observed the call (machine too fast or too loaded). A late success is a + probe that succeeded only after the call returned: it is not a failure, but its effect stays (an + append left one more element), so callers that check sizes add it back (#401).""" for _ in range(attempts): go, done = threading.Event(), threading.Event() - seen = {name: [0, []] for name in probes} + seen = {name: [0, [], 0] for name in probes} def worker(): go.wait() @@ -489,7 +500,14 @@ def worker(): except Exception as e: # noqa: BLE001 seen[name][1].append(repr(e)) else: - seen[name][1].append("succeeded") + # The call can return between the `done` check above and this probe; + # the buffer is then legitimately free. A probe that succeeds while the + # buffer is really unheld does so ~0.05 s into a ~0.6 s call, so `done` + # stays unset for far longer than this grace period. + if not done.wait(0.05): + seen[name][1].append("succeeded") + else: + seen[name][2] += 1 return time.sleep(0.001) @@ -519,7 +537,7 @@ def test_other_thread_release_and_append_get_bufferror_during_a_memoryview_call( assert result == float(n) assert seen["release"][0] > 0 and seen["release"][1] == [], seen assert seen["append"][0] > 0 and seen["append"][1] == [], seen - assert len(a) == N # nothing resized under the call + assert len(a) == N + seen["append"][2] # nothing resized under the call (#401) assert release_ok(v) and can_resize(a) # held only during the call @@ -533,7 +551,8 @@ def test_other_thread_append_gets_bufferror_during_an_array_call(built): assert result == float(n) for name in ("append", "extend", "pop"): assert seen[name][0] > 0 and seen[name][1] == [], (name, seen) - assert len(a) == N + # nothing resized under the call; a probe that succeeded after it returned stays applied (#401) + assert len(a) == N + seen["append"][2] + seen["extend"][2] - seen["pop"][2] assert can_resize(a) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py index 582e4b094..f14a32c00 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py @@ -21,13 +21,13 @@ from test_buffer_arrays import build_source, load_so # noqa: E402 SOURCE = textwrap.dedent(''' - # typedpython: compiled import array import math import numpy as np import numpy.typing as npt + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -35,18 +35,22 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def nd_first(x: npt.NDArray[np.float64]) -> object: return x[0] + @compiled def nd_first_f(x: npt.NDArray[np.float64]) -> float: return x[0] + @compiled def nd_first_i(x: npt.NDArray[np.int64]) -> int: return x[0] + @compiled def nd_sum(x: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -54,6 +58,7 @@ def nd_sum(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: s = x[0] for i in range(1, len(x)): @@ -61,6 +66,7 @@ def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_isum(x: npt.NDArray[np.int64]) -> int: s = 0 for i in range(len(x)): @@ -68,64 +74,79 @@ def nd_isum(x: npt.NDArray[np.int64]) -> int: return s + @compiled def nd_gt(x: npt.NDArray[np.float64]) -> object: return x[0] > x[1] + @compiled def nd_flag(x: npt.NDArray[np.float64]) -> object: b = x[0] > 0.0 return b + @compiled def nd_plain(x: npt.NDArray[np.float64]) -> object: return float(x[0]) + @compiled def nd_mixed(x: npt.NDArray[np.float64], k: float) -> object: return k + 1.0 + @compiled def nd_sqrt(x: npt.NDArray[np.float64]) -> object: return math.sqrt(x[0] * x[0]) + @compiled def nd_neg(x: npt.NDArray[np.float64]) -> object: return -x[1] + @compiled def nd_len(x: npt.NDArray[np.float64]) -> int: return len(x) + @compiled def nd_set(x: npt.NDArray[np.float64], i: int, v: float) -> None: x[i] = v + @compiled def nd_copy(dst: npt.NDArray[np.float64], src: npt.NDArray[np.float64]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def nd_iset(x: npt.NDArray[np.int64], i: int, v: int) -> None: x[i] = v + @compiled def nd_get(x: npt.NDArray[np.float64], i: int) -> object: return x[i] + @compiled def nd_overflow(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def nd_invalid(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def nd_iover(x: npt.NDArray[np.int64]) -> object: return x[0] + x[1] + @compiled def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> float: @@ -135,6 +156,7 @@ def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDA return e + @compiled def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> None: @@ -145,20 +167,24 @@ def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDAr m[i] = x[i] + @compiled def nd_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def nd_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def nd_param(x: npt.NDArray[np.float64], k: float, use: bool) -> object: if use: k = x[0] return k + @compiled def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: if use: v = x[0] @@ -167,36 +193,44 @@ def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: return v + @compiled def nd_tuple(x: npt.NDArray[np.int64]) -> tuple[int, int, bool]: return (x[0], 3, x[1] > 0) + @compiled def nd_abs(x: npt.NDArray[np.float64]) -> object: return abs(x[1]) + @compiled def nd_to_list(x: npt.NDArray[np.float64], dst: list[float]) -> None: dst[0] = x[0] + @compiled def nd_mixed_cmp(x: npt.NDArray[np.int64], f: float) -> object: if x[0] < f: return 1 return 0 + @compiled def whole_sum(x: npt.NDArray[np.float64]) -> object: return np.sum(x) + @compiled def whole_attr(x: npt.NDArray[np.float64]) -> object: return x.size + @compiled def whole_pass(x: npt.NDArray[np.float64]) -> object: return np.add(x, 1.0).sum() + @compiled def nd_int_impure(x: npt.NDArray[np.int64]) -> object: s = 0 for i in range(len(x)): @@ -205,83 +239,103 @@ def nd_int_impure(x: npt.NDArray[np.int64]) -> object: return s + @compiled def nd_scale(x: npt.NDArray[np.float64], k: float) -> None: for i in range(len(x)): x[i] = x[i] * k + @compiled def p_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def p_fdiv(x: npt.NDArray[np.float64]) -> object: return x[0] // x[1] + @compiled def p_mod(x: npt.NDArray[np.float64]) -> object: return x[0] % x[1] + @compiled def p_add(x: npt.NDArray[np.float64]) -> object: return x[0] + x[1] + @compiled def p_sub(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def p_mul(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def p_pdiv(x: npt.NDArray[np.float64], k: float) -> object: return k / x[0] + @compiled def p_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] / x[1] + @compiled def p_ifdiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def p_imod(x: npt.NDArray[np.int64]) -> object: return x[0] % x[1] + @compiled def p_imixed(x: npt.NDArray[np.int64]) -> object: return x[0] // 3 + x[1] % 4 + @compiled def i_div(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] / x[1] + @compiled def i_fdiv(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] // x[1] + @compiled def i_mod(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] % x[1] + @compiled def i_add(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] + x[1] + @compiled def i_sub(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] - x[1] + @compiled def i_mul(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] * x[1] + @compiled def i_plain(x: npt.NDArray[np.float64], k: float) -> None: x[2] = k / x[1] x[3] = k * 2.0 + @compiled def i_multiline(x: npt.NDArray[np.float64]) -> None: x[2] = (x[0] * x[1]) @@ -290,12 +344,14 @@ def i_multiline(x: npt.NDArray[np.float64]) -> None: + x[0] * x[1]) + @compiled def i_aug(x: npt.NDArray[np.float64]) -> None: x[0] *= x[1] x[1] /= x[2] x[2] += x[3] + @compiled def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -304,21 +360,25 @@ def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: return s + @compiled def i_mixed(x: npt.NDArray[np.float64], n: int) -> None: x[0] = n / x[1] x[2] = x[3] * n + @compiled def i_ret(x: npt.NDArray[np.float64]) -> object: x[2] = x[0] * x[1] return x[0] / x[1] + @compiled def i_ifdiv(x: npt.NDArray[np.int64]) -> object: x[2] = x[0] // x[1] return x[2] + @compiled def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -343,6 +403,7 @@ def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray z[i] += dt * vz[i] + @compiled def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -357,10 +418,12 @@ def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) return e # #261: np.bool_ arrays (1-byte items, format "?") + @compiled def nd_b_get(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + @compiled def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: for i in range(len(x)): if x[i]: @@ -368,6 +431,7 @@ def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: return False + @compiled def nd_b_count(x: npt.NDArray[np.bool_]) -> int: c = 0 for i in range(len(x)): @@ -376,45 +440,55 @@ def nd_b_count(x: npt.NDArray[np.bool_]) -> int: return c + @compiled def nd_b_set(x: npt.NDArray[np.bool_], i: int, v: bool) -> None: x[i] = v + @compiled def nd_b_flip(x: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): x[i] = not x[i] + @compiled def nd_b_copy(a: npt.NDArray[np.bool_], b: npt.NDArray[np.bool_]) -> None: for i in range(len(a)): b[i] = a[i] + @compiled def nd_b_eq(x: npt.NDArray[np.bool_], y: npt.NDArray[np.bool_], i: int) -> object: return x[i] == y[i] + @compiled def nd_b_gt(x: npt.NDArray[np.float64], m: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): m[i] = x[i] > 2.0 + @compiled def nd_b_len(x: npt.NDArray[np.bool_]) -> int: return len(x) + @compiled def nd_b_store_int(x: npt.NDArray[np.bool_], i: int, v: int) -> None: x[i] = v + @compiled def nd_b_store_float(x: npt.NDArray[np.bool_], i: int, v: float) -> None: x[i] = v + @compiled def nd_b_arith(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + 1 + @compiled def nd_b_whole(x: npt.NDArray[np.bool_]) -> object: return np.sum(x) ''') diff --git a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py index 4f184812b..1357c7ed7 100644 --- a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py +++ b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py @@ -21,7 +21,16 @@ def test_gate_accepts_compiled_without_an_import(write): def f(x: int) -> int: return x + 1 - y: int = f(1) + @compiled(parallel=True) + def g(x: int) -> int: + return x + 2 + + @compiled(fastmath=True) + def h(x: float) -> float: + return x * 2.0 + + y: int = f(1) + g(2) + z: float = h(1.5) """) assert errors(gate.check([path])) == [] @@ -82,7 +91,7 @@ def test_decorator_works_with_no_import(installed): def test_compiled_is_the_identity(installed): python, _ = installed - r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f)") + r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f and compiled(parallel=True)(f) is f and compiled()(f) is f)") assert r.stdout.strip() == "True", r.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py index dfbeb0058..4858212dc 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py @@ -428,6 +428,20 @@ def test_gen_uses_only_the_runtime_helpers_for_slots(): assert "PyMember_" not in text and "tp_alloc" not in text and "->tp_dictoffset" not in text +def test_new_fixed_fills_its_fresh_object_with_direct_release_stores(): + """#125: a New allocates its object and fills every slot before control can reach a safepoint, so + no other thread holds the object yet. tp_new_fixed stores each new reference straight into the + slot (release store, as PyMember_SetOne's free-threaded store) instead of taking the member + protocol's per-object critical section for every slot.""" + from typedpython import cbuild + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + start = h.index("static inline PyObject *tp_new_fixed(") + body = h[start:h.index("\n}\n", start)] + assert "PyMember_SetOne" not in body and "CRITICAL_SECTION" not in body, body + assert "__atomic_store_n(" in body and "__ATOMIC_RELEASE" in body, body + assert "Py_NewRef(" in body, body # the slot owns its own reference + + # --- refcounts --------------------------------------------------------------------------------------------- def test_reference_counts_are_unchanged_over_many_calls(m): diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index d6693dcad..2957adb8a 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -40,6 +40,7 @@ import pytest +from conftest import COMPILED_SHIM, decorate_compiled from typedpython import cbuild, cgen, cost, frontend, pipeline, verify HERE = Path(__file__).resolve().parent @@ -51,10 +52,10 @@ BINARY_TREES = REPO / "benchmarks" / "typedpython" / "py" / "binary_trees.py" SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython class-method test module.""" + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -135,21 +136,25 @@ def no_annotation(self, x) -> int: return 1 + @compiled class Plain: def check(self) -> int: return 5 + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check_of(n: Node) -> int: return n.check() + @compiled def leaf_of(n: Node) -> bool: return n.leaf() ''') @@ -231,6 +236,7 @@ def test_the_methods_of_a_class_that_is_not_fixed_layout_are_not_considered(tmp_ def test_a_method_named_like_a_module_function_does_not_collide(tmp_path): m = lower(tmp_path, SOURCE + textwrap.dedent('''\ + @compiled def height(n: Node) -> int: return n.height() ''')) @@ -240,7 +246,7 @@ def height(n: Node) -> int: def test_a_method_with_a_foreign_self_annotation_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -258,7 +264,7 @@ def get(self: int) -> int: def test_a_method_defined_twice_in_the_class_body_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -277,7 +283,7 @@ def get(self) -> int: assert "more than once" in m.skipped["A.get"] -def test_an_at_compiled_class_compiles_its_methods_without_the_module_marker(tmp_path): +def test_an_at_compiled_class_compiles_its_methods(tmp_path): src = textwrap.dedent('''\ @compiled class A: @@ -547,7 +553,7 @@ def test_class_info_reports_the_methods(built): # --- class state changed after import: child processes -------------------------------------------- -CHILD = ''' +CHILD = COMPILED_SHIM + ''' import json, sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -730,7 +736,7 @@ def test_reference_counts_are_flat_over_many_calls(built): # --- free-threaded concurrency -------------------------------------------------------------------- -THREADS = ''' +THREADS = COMPILED_SHIM + ''' import sys, threading from pathlib import Path sys.path.insert(0, {main!r}) @@ -794,7 +800,7 @@ def test_concurrent_calls_on_shared_and_private_instances(built): # --- mutation: the exact-type guard of `self` ------------------------------------------------------- -GUARD_PROBE = ''' +GUARD_PROBE = COMPILED_SHIM + ''' import sys from pathlib import Path sys.path.insert(0, {main!r}) @@ -887,7 +893,7 @@ def run_main(path_or_module, n): def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): - text = "# typedpython: compiled\n" + method_form(BINARY_TREES.read_text()) + text = decorate_compiled(method_form(BINARY_TREES.read_text())) path = tmp_path / "bt_method.py" path.write_text(text) result = pipeline.compile_module(path, tmp_path / "out") @@ -914,15 +920,17 @@ def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): "fannkuch": {"fannkuch"}, "spectral_norm": {"eval_a", "mul_atv", "mul_av"}, "wordfreq": {"next_state", "word_of", "main"}, # word_of and main: native ints, #232 - "nbody": set(), + # #255: a pure function reads its three or more lists element by element (energy); advance and + # offset_momentum store into theirs and stay interpreted (no lock for more than two objects) + "nbody": {"energy"}, } @pytest.mark.parametrize("name", sorted(EXPECTED)) def test_the_benchmarks_compile_the_same_functions_as_before(tmp_path, name): path = tmp_path / f"{name}.py" - path.write_text("# typedpython: compiled\n" + (REPO / "benchmarks" / "typedpython" / "py" - / f"{name}.py").read_text()) + path.write_text(decorate_compiled((REPO / "benchmarks" / "typedpython" / "py" + / f"{name}.py").read_text())) proved, diags = verify.verify(frontend.lower(path)) proved, diags = cost.apply(proved, diags) assert {f.name for f in proved.functions} == EXPECTED[name] diff --git a/python-multiplatform-ksp/src/test/python/test_cost_rule.py b/python-multiplatform-ksp/src/test/python/test_cost_rule.py index db6c4999a..c540bb260 100644 --- a/python-multiplatform-ksp/src/test/python/test_cost_rule.py +++ b/python-multiplatform-ksp/src/test/python/test_cost_rule.py @@ -12,20 +12,19 @@ import pytest +from conftest import decorate_compiled from typedpython import cost, pipeline -HEADER = "# typedpython: compiled\n" - -OBJECT_ONLY = HEADER + textwrap.dedent(''' +OBJECT_ONLY = decorate_compiled(textwrap.dedent(''' def copy_words(src: list[str], dst: list[str]) -> str: w: str = "" for x in src: w = x + w dst.append(w) return w -''') +''')) -INT_LOOP = HEADER + textwrap.dedent(''' +INT_LOOP = decorate_compiled(textwrap.dedent(''' def word_of(word_id: int) -> str: x: int = word_id + 17576 w: str = "" @@ -33,7 +32,7 @@ def word_of(word_id: int) -> str: w = chr(97 + x % 26) + w x //= 26 return w -''') +''')) def build(tmp_path, source, **kw): @@ -67,7 +66,7 @@ def test_the_opt_out_compiles_the_same_function(tmp_path): assert names(r) == {"copy_words"} and r.extension is not None -MIXED = HEADER + textwrap.dedent(''' +MIXED = decorate_compiled(textwrap.dedent(''' def mixed(n: int, xs: list[str]) -> float: total: float = 0.0 for i in range(n): @@ -106,7 +105,7 @@ def native_loop_inside_object_loop(rows: list[list[str]]) -> float: for k in range(3): total += 0.5 * 2.0 return total -''') +''')) def test_one_native_loop_with_object_operations_compiles(tmp_path): @@ -148,10 +147,10 @@ def test_the_rule_is_what_keeps_it_interpreted(tmp_path, monkeypatch): test_an_object_only_loop_stays_interpreted_with_the_reason(tmp_path) -STRAIGHT_LINE = HEADER + textwrap.dedent(''' +STRAIGHT_LINE = decorate_compiled(textwrap.dedent(''' def label(n: int, name: str) -> str: return f"{name}:{n + 1}" -''') +''')) def test_a_straight_line_impure_function_still_compiles(tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_depth_once.py b/python-multiplatform-ksp/src/test/python/test_depth_once.py new file mode 100644 index 000000000..64e02af2b --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_depth_once.py @@ -0,0 +1,546 @@ +"""TypedPython: the Python frame depth is measured once per segment, not once per compiled frame +(#125, docs/design/typedpython.md 4.3.3b, agent proposal, not confirmed). + +A *segment* is one call of a compiled impl from its wrapper (`tp_wrap_f`), up to the first point +where a call of a wrapper of the same module returns inside it. Inside a segment the Python frames +below the compiled frames cannot change: Python code that runs in between (a callback, a `__del__`, +a signal handler) pushes frames and pops all of them before it returns to C. So the first depth check +of a segment measures the Python depth (`tp_python_depth`, one `PyThreadState_GetFrame`) and every +later check of the segment reuses it. The wrapper resets the measurement before and after its impl +call, so a nested wrapper never sees the outer segment's depth and the outer segment measures again +after a nested wrapper returned (exactly where it measured before this change, so the cached frame +object and every frame's lifetime stay the same). + +What must NOT move, and stays per frame: `Py_GetRecursionLimit()` (a `__del__` or a callback can call +`sys.setrecursionlimit` between two compiled frames) and `Py_EnterRecursiveCall` (the C-stack guard: +it compares this frame's stack address). + + * red before the implementation: `__tp_trace_frame_reads__` (-DTP_TRACE_DEPTH) counts one + `PyThreadState_GetFrame` per compiled frame (`down(100)`: 101 instead of 1), and the generated + wrappers have no `tp_depth_seg_begin` / `tp_depth_seg_end`; + * green before and after (they pin what must not change): every differential test against the + interpreter (depth at every Python depth and recursion depth around the limit, a nested compiled + call from a callback, the limit changed by a `__del__` and by a callback mid-recursion, threads), + the frame-lifetime test and the `sys._getframe` test; + * mutants: a wrapper that does not reset before its impl, one that does not reset after it, and a + limit read once per call instead of per frame must each be caught. + +Builds go through the front end, verifier, cgen and cbuild in pytest's tmp_path. +""" +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import COMPILED_SHIM +from typedpython import cbuild, cgen, frontend, verify + +SOURCE = """\ +from typing import Callable + + +@compiled +def down(n: int) -> float: + if n == 0: + return 0.0 + return 1.0 + down(n - 1) + + +@compiled +def visit(a: Callable[[int], object], b: Callable[[int], object], n: int) -> None: + a(n) + b(n) + if n > 0: + visit(a, b, n - 1) + + +@compiled +def around(cb: Callable[[], None], m: int) -> float: + cb() + return down(m) + + +@compiled +def seq(g: Callable[[], None], probe: Callable[[], None], m: int) -> float: + g() + probe() + r = down(m) + probe() + return r + + +@compiled +def leaf(i: int) -> float: + return 1.0 / (i + 1) + + +@compiled +def mid(i: int) -> float: + return leaf(i) + leaf(i + 1) + + +@compiled +def deep_mid(n: int) -> float: + if n == 0: + return mid(0) + return 1.0 + deep_mid(n - 1) +""" + +NAMES = {"down", "visit", "around", "seq", "leaf", "mid", "deep_mid"} + + +def generate(d: Path): + path = d / "depth_mod.py" + path.write_text(SOURCE) + proved, _ = verify.verify(frontend.lower(path)) + return path, proved, cgen.generate(proved, path, path.name) + + +def build(d: Path, extra: tuple[str, ...] = (), runtime_dir: Path | None = None): + path, proved, c = generate(d) + so = cbuild.build(c, path.stem, d / "out", runtime_dir=runtime_dir, + flags=cbuild.DEFAULT_FLAGS + extra) + return proved, c, so + + +def mutant_runtime(tmp_path: Path, *edits: tuple[str, str]) -> Path: + mutant = tmp_path / "runtime" + shutil.copytree(cbuild.RUNTIME_DIR, mutant) + text = (mutant / "tp_runtime.h").read_text() + for old, new in edits: + assert text.count(old) == 1, old + text = text.replace(old, new) + (mutant / "tp_runtime.h").write_text(text) + return mutant + + +@pytest.fixture(scope="module") +def generated(tmp_path_factory): + return generate(tmp_path_factory.mktemp("depth_c")) + + +@pytest.fixture(scope="module") +def plain_build(tmp_path_factory): + return build(tmp_path_factory.mktemp("depth_plain")) + + +@pytest.fixture(scope="module") +def plain_src(tmp_path_factory): + p = tmp_path_factory.mktemp("depth_src") / "depth_mod_plain.py" + p.write_text(SOURCE) + return p + + +def fn_text(c: str, signature_start: str) -> str: + """The text of the C function whose definition starts with `signature_start`.""" + i = c.index(signature_start + "(") + while not c[c.index(")", i) + 1:].startswith("\n{"): + i = c.index(signature_start + "(", i + 1) + return c[i:c.index("\n}\n", i)] + + +# --- generated C and the runtime header ---------------------------------------------------------- + +def test_every_function_compiles(generated): + proved = generated[1] + assert {f.name for f in proved.functions} == NAMES, proved.skipped + + +def test_every_wrapper_resets_the_segment_right_around_its_impl_call(generated): + c = generated[2] + for name in NAMES: + w = fn_text(c, f"static PyObject *tp_wrap_{name}").splitlines() + calls = [i for i, line in enumerate(w) if re.match(rf"\s*tp_s = tp_impl_{name}\(", line)] + assert len(calls) == 1, (name, calls) + i = calls[0] + assert w[i - 1].strip() == "tp_depth_seg_begin(tp_t);", (name, w[i - 1]) + assert w[i + 1].strip() == "tp_depth_seg_end(tp_t);", (name, w[i + 1]) + assert sum("tp_depth_seg_" in line for line in w) == 2, name + + +def test_impls_never_reset_or_measure_the_segment_themselves(generated): + c = generated[2] + for name in NAMES: + for sig in (f"static int tp_impl_{name}", f"static inline int tp_fimpl_{name}"): + if sig + "(" not in c: + continue + body = fn_text(c, sig) + assert "tp_depth_seg_" not in body and "tp_python_depth" not in body, sig + assert "PyThreadState_GetFrame" not in body, sig + + +def header_fn(name: str) -> str: + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + i = re.search(rf"static inline [^\n(]*\b{name}\(", h).start() + return h[i:h.index("\n}\n", i)] + + +def test_the_frame_checks_use_the_segment_depth_and_keep_the_per_frame_calls(): + enter = header_fn("tp_enter_call") + room = header_fn("tp_depth_room") + for body in (enter, room): + assert "tp_seg_depth(ts, t)" in body and "tp_python_depth" not in body, body + assert "Py_GetRecursionLimit()" in body, body # the limit stays per frame + assert "Py_EnterRecursiveCall(" in enter # the C-stack guard stays per frame + seg = header_fn("tp_seg_depth") + assert "tp_python_depth(ts, t)" in seg and "seg < 0" in seg + + +# --- children: differential runs against the interpreter ----------------------------------------- + +LOAD = COMPILED_SHIM + """\ +import importlib.machinery, importlib.util, json, sys +def load(name, path, ext): + if ext: + loader = importlib.machinery.ExtensionFileLoader(name, path) + spec = importlib.util.spec_from_file_location(name, path, loader=loader) + else: + spec = importlib.util.spec_from_file_location(name, path) + m = importlib.util.module_from_spec(spec) + (loader if ext else spec.loader).exec_module(m) + return m +comp = load("depth_mod", sys.argv[1], True) +plain = load("depth_mod_plain", sys.argv[2], False) +LIMIT = int(sys.argv[3]) +sys.setrecursionlimit(LIMIT) + +def descend(d, fn, a): + if d > 0: + return descend(d - 1, fn, a) + try: + return ["ok", fn(a)] + except RecursionError: + return ["rec"] + +def run(fn, a, d): + try: + return descend(d, fn, a) + except RecursionError: + return ["deep"] + +report = {"mismatch": [], "kinds": {}} +def note(tag, c, p): + if c != p: + report["mismatch"].append([tag, c, p]) + k = report["kinds"].setdefault(tag, {}) + key = c[0] if isinstance(c[0], str) else c[0][0] + k[key] = k.get(key, 0) + 1 + +inner = [] +def mk(mod, k, j): + # a Python callback that descends k Python frames, then makes a nested compiled call down(j) + def cb(): + def dive(i): + if i == 0: + try: + inner.append(["ok", mod.down(j)]) + except RecursionError: + inner.append(["rec"]) + return + dive(i - 1) + dive(k) + return cb + +def nested(mod, d, k, j, m): + del inner[:] + r = run(lambda a: mod.around(mk(mod, k, j), a), m, d) + return [r, list(inner)] +""" + +CHILD_SWEEP = LOAD + """\ +for d in range(1, LIMIT + 6): # every Python depth around the limit + note("down@d", run(comp.down, 20, d), run(plain.down, 20, d)) +for n in range(LIMIT - 40, LIMIT + 6): # every recursion depth around the limit + note("down@n", run(comp.down, n, 5), run(plain.down, n, 5)) + # deep counted recursion, then a bounded callee whose precheck (tp_depth_room) reuses the depth + note("deep_mid", run(comp.deep_mid, n, 5), run(plain.deep_mid, n, 5)) +print(json.dumps(report)) +""" + +CHILD_NESTED = LOAD + """\ +# inner: a nested wrapper inside a segment must measure its own (deeper) Python depth +for j in range(LIMIT - 70, LIMIT + 6): + note("inner", nested(comp, 10, 30, j, 0), nested(plain, 10, 30, j, 0)) +# outer: after the nested wrapper returned, the outer segment must not keep the inner depth +for m in range(LIMIT - 70, LIMIT + 6): + note("outer", nested(comp, 10, 30, 0, m), nested(plain, 10, 30, 0, m)) +print(json.dumps(report)) +""" + +CHILD_LIMIT = LOAD + """\ +class Setter: + def __init__(self, to): + self.to = to + def __del__(self): + sys.setrecursionlimit(self.to) + +def by_del(mod, d, n, at, to): + # visit(a, b, n) calls a(n) then b(n) in every frame; b = objs.pop drops the last element, and + # at n == at that is a Setter whose only reference is the compiled call's result: its __del__ + # runs when the compiled frame releases it (the interpreter: POP_TOP), between two frames + sys.setrecursionlimit(LIMIT) + log = [] + objs = [None] * (n + 1) + objs[at] = Setter(to) + r = run(lambda a: mod.visit(log.append, objs.pop, a), n, d) + sys.setrecursionlimit(LIMIT) + return [r, len(log)] + +def by_callback(mod, d, m, to): + sys.setrecursionlimit(LIMIT) + r = run(lambda a: mod.around(lambda: sys.setrecursionlimit(to), a), m, d) + sys.setrecursionlimit(LIMIT) + return [r] + +N = 2 * LIMIT +for to in range(60, LIMIT - 20, 3): # lowered 20 frames in: the next frames see it + note("del-lower", by_del(comp, 5, N, N - 20, to), by_del(plain, 5, N, N - 20, to)) +for to in (LIMIT + 40, 3 * LIMIT): # raised 20 frames in: the recursion may go deeper + note("del-raise", by_del(comp, 5, N, N - 20, to), by_del(plain, 5, N, N - 20, to)) +for m in range(40, 140, 1): # a callback lowers it before a compiled recursion + note("cb-lower", by_callback(comp, 5, m, 100), by_callback(plain, 5, m, 100)) +print(json.dumps(report)) +""" + +CHILD_FRAMES = LOAD + """\ +events = [] + +class Obj: + def __del__(self): + events.append("del") + +def mkg(mod): + def g(): + x = Obj() # dies with g's frame (or with the frame object that keeps g's locals) + mod.down(1) + return g + +def probe(): + events.append("probe") + +out = {} +del events[:] +comp.seq(mkg(comp), probe, 3) +out["compiled"] = list(events) +del events[:] +plain.seq(mkg(plain), probe, 3) +out["interpreted"] = list(events) + +def chain(): + f, names = sys._getframe(2), [] # from the callback's caller down + while f is not None: + names.append(f.f_code.co_name) + f = f.f_back + return names + +seen = [] +def cb(): + seen.append(chain()) +def cb_nested(): + comp.down(3) # a nested wrapper first: its segment must not change what cb sees + seen.append(chain()) +def cb_nested_plain(): + plain.down(3) + seen.append(chain()) + +def at(d, fn): + if d > 0: + return at(d - 1, fn) + return fn() + +for d in (0, 7, 40): + at(d, lambda: comp.around(cb, 5)) + at(d, lambda: comp.around(cb_nested, 5)) + at(d, lambda: comp.around(cb, 5)) + at(d, lambda: plain.around(cb, 5)) + at(d, lambda: plain.around(cb_nested_plain, 5)) +out["chains"] = seen +print(json.dumps(out)) +""" + +CHILD_THREADS = LOAD + """\ +import threading +results = {} +def worker(t): + mine = [] + for j in range(LIMIT - 60, LIMIT + 6, 2): + mine.append([nested_t(comp, 3 + t, 10 + t, j, 0), nested_t(plain, 3 + t, 10 + t, j, 0)]) + mine.append([nested_t(comp, 3 + t, 10 + t, 0, j), nested_t(plain, 3 + t, 10 + t, 0, j)]) + results[t] = mine + +local = threading.local() +def mk_t(mod, k, j): + def cb(): + def dive(i): + if i == 0: + try: + local.inner.append(["ok", mod.down(j)]) + except RecursionError: + local.inner.append(["rec"]) + return + dive(i - 1) + dive(k) + return cb + +def nested_t(mod, d, k, j, m): + local.inner = [] + r = run(lambda a: mod.around(mk_t(mod, k, j), a), m, d) + return [r, list(local.inner)] + +ts = [threading.Thread(target=worker, args=(t,)) for t in range(4)] +[t.start() for t in ts] +[t.join() for t in ts] +for t, rows in sorted(results.items()): + for c, p in rows: + note(f"thread{t}", c, p) +print(json.dumps(report)) +""" + + +def run_child(script: str, so: Path, plain_src: Path, limit: str = "200") -> dict: + proc = subprocess.run([sys.executable, "-c", script, str(so), str(plain_src), limit], + capture_output=True, text=True, timeout=600) + assert proc.returncode == 0, proc.stdout + proc.stderr + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +def assert_sweep(r): + assert r["mismatch"] == [], r["mismatch"][:5] + for tag in ("down@d", "down@n", "deep_mid"): # the sweep crossed the boundary + assert r["kinds"][tag].get("ok", 0) > 5 and r["kinds"][tag].get("rec", 0) >= 1, (tag, r["kinds"]) + + +def assert_nested(r): + assert r["mismatch"] == [], r["mismatch"][:5] + for tag in ("inner", "outer"): + assert r["kinds"][tag].get("ok", 0) > 5, (tag, r["kinds"]) + # crossing the boundary: some outer runs end in RecursionError, some inner calls do too + assert r["kinds"]["outer"].get("rec", 0) >= 1, r["kinds"] + + +def assert_limit(r): + assert r["mismatch"] == [], r["mismatch"][:5] + assert r["kinds"]["del-lower"].get("rec", 0) >= 10, r["kinds"] # the lowered limit stopped them + assert r["kinds"]["del-raise"].get("ok", 0) >= 1, r["kinds"] # the raised one let one through + k = r["kinds"]["cb-lower"] + assert k.get("ok", 0) > 5 and k.get("rec", 0) > 5, k + + +def test_depth_at_every_python_depth_and_recursion_depth_matches_the_interpreter(plain_build, plain_src): + assert_sweep(run_child(CHILD_SWEEP, plain_build[2], plain_src)) + + +def test_a_nested_compiled_call_from_a_callback_matches_the_interpreter(plain_build, plain_src): + assert_nested(run_child(CHILD_NESTED, plain_build[2], plain_src)) + + +def test_a_limit_changed_by_a_del_or_a_callback_mid_recursion_is_honoured(plain_build, plain_src): + assert_limit(run_child(CHILD_LIMIT, plain_build[2], plain_src, "300")) + + +def test_threads_each_see_their_own_depth(plain_build, plain_src): + r = run_child(CHILD_THREADS, plain_build[2], plain_src) + assert r["mismatch"] == [], r["mismatch"][:5] + assert sorted(r["kinds"]) == [f"thread{t}" for t in range(4)], r["kinds"] + + +def test_frame_lifetimes_and_getframe_are_unchanged(plain_build, plain_src): + """Pins develop's behaviour (green before and after the change). + + Lifetime: inside `seq`, the nested `down(1)` called from the Python callback g measures the depth + with g's frame on top, so the depth cache holds g's frame object; when g returns, that frame + object takes g's locals (`take_ownership`, Python/frame.c), and `x` dies when the outer segment + next measures, at its `down(m)`, not at g's return as in the interpreter. That difference from the + interpreter exists on develop (see the design note); this test only pins that the change keeps + the same moment: a wrapper that kept the inner segment's depth after it returned would never + re-measure and `x` would live until `seq` returns (["probe", "probe", "del"]). + + sys._getframe: the frames a callee sees are the Python frames only (compiled frames have none), + the same at every depth and after a nested compiled call.""" + r = run_child(CHILD_FRAMES, plain_build[2], plain_src) + assert r["interpreted"] == ["del", "probe", "probe"], r + assert r["compiled"] == ["probe", "del", "probe"], r + chains = r["chains"] + assert len(chains) == 15 + for i in range(3): + c1, c2, c3, p1, p2 = chains[5 * i:5 * i + 5] + assert c1 == c2 == c3, (c1, c2, c3) + assert c1 == [n for n in p1 if n not in ("around", "down")], (c1, p1) + assert p1 == p2 + assert len(chains[10]) > len(chains[0]) # deeper at d=40 than at d=0 + + +# --- the measurement really happens once per segment (red before the implementation) ------------- + +def test_one_frame_read_per_segment(tmp_path): + proved, c, so = build(tmp_path, ("-DTP_TRACE_DEPTH",)) + m = cbuild.load("depth_mod", so) + + def reads(fn): + before = m.__tp_trace_frame_reads__() + fn() + return m.__tp_trace_frame_reads__() - before + + # develop measured once per compiled frame: 101, 52 (51 frames + mid's precheck), 1, 101, 18 + assert reads(lambda: m.down(100)) == 1 + assert reads(lambda: m.deep_mid(50)) == 1 + assert reads(lambda: m.mid(3)) == 1 + log: list = [] + objs = [None] * 101 + assert reads(lambda: m.visit(log.append, objs.pop, 100)) == 1 and len(log) == 101 + # around: its own segment, the nested down(5) from the callback, then around measures again + # after that wrapper returned (its down(10) is the first check after the nested segment) + assert reads(lambda: m.around(lambda: m.down(5), 10)) == 3 + + +def test_the_trace_counter_is_absent_from_a_normal_build(plain_build): + c, so = plain_build[1], plain_build[2] + assert "TP_TRACE_DEPTH" in c # guarded, so compiled out + m = cbuild.load("depth_mod", so) + assert not hasattr(m, "__tp_trace_frame_reads__") + + +# --- mutants: each reset and the per-frame limit read are shown to matter ------------------------- + +SEG_BEGIN = "static inline void tp_depth_seg_begin(tp_tls *t) { t->depth.seg = -1; }" +SEG_END = "static inline void tp_depth_seg_end(tp_tls *t) { t->depth.seg = -1; }" +ENTER = "static inline int tp_enter_call(PyThreadState *ts, tp_tls *t)\n{" +ENTER_CHECK = "if (tp_seg_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) {" + + +def test_a_wrapper_that_does_not_reset_before_its_impl_is_caught(tmp_path, plain_src): + """Mutation: a nested wrapper reuses the outer segment's (shallower) depth.""" + rt = mutant_runtime(tmp_path, (SEG_BEGIN, "static inline void tp_depth_seg_begin(tp_tls *t) { (void)t; }")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_nested(run_child(CHILD_NESTED, so, plain_src)) + + +def test_a_wrapper_that_does_not_reset_after_its_impl_is_caught(tmp_path, plain_src): + """Mutation: the outer segment keeps the inner segment's (deeper) depth and never re-measures.""" + rt = mutant_runtime(tmp_path, (SEG_END, "static inline void tp_depth_seg_end(tp_tls *t) { (void)t; }")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_nested(run_child(CHILD_NESTED, so, plain_src)) + r = run_child(CHILD_FRAMES, so, plain_src) + assert r["compiled"] == ["probe", "probe", "del"], r # and x lives longer + + +def test_a_limit_read_once_per_call_is_caught(tmp_path, plain_src): + """Mutation: the recursion limit is read at the first compiled frame only (count == 0), as a + once-per-wrapper limit would be. A limit changed mid-recursion is then missed.""" + rt = mutant_runtime( + tmp_path, + (ENTER, "static int tp_mut_limit = 0;\n" + ENTER), + (ENTER_CHECK, "if (t->depth.count == 0) tp_mut_limit = Py_GetRecursionLimit();\n" + " if (tp_seg_depth(ts, t) + t->depth.count + 1 > tp_mut_limit) {")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_limit(run_child(CHILD_LIMIT, so, plain_src, "300")) diff --git a/python-multiplatform-ksp/src/test/python/test_emit_c.py b/python-multiplatform-ksp/src/test/python/test_emit_c.py new file mode 100644 index 000000000..737137899 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_emit_c.py @@ -0,0 +1,121 @@ +"""`python -m typedpython emit-c` (#25 step 1, SPEC N-10): generate_project plus a JSON manifest that +maps onto pypackpack's BundleRequest.nativeModules / notNativeModules. Format: docs/design/typedpython.md +section 4.3.2a. + +Real Pyrefly; the C compiler is never run here. +""" +import json +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild +from typedpython.__main__ import main + +A = decorate_compiled(textwrap.dedent("""\ + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" +NOTHING = "@compiled\ndef f(s: str) -> str:\n return s[1:]\n" +BROKEN = "@compiled\ndef f(n: int) -> int:\n return n + 'x'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "app").mkdir(parents=True) + (root / "app" / "__init__.py").write_text("") + (root / "app" / "fast.py").write_text(A) + (root / "app" / "plain.py").write_text(PLAIN) + (root / "app" / "text.py").write_text(NOTHING) + return root + + +@pytest.fixture(autouse=True) +def no_c_compiler(monkeypatch): + def build(*a, **k): + raise AssertionError("emit-c must not run the C compiler") + monkeypatch.setattr(cbuild, "build", build) + + +def emit(project, tmp_path, *paths): + manifest = tmp_path / "out" / "manifest.json" + rc = main(["emit-c", *(str(p) for p in (paths or [project])), "--root", str(project), + "--cache", str(tmp_path / "cache"), "--manifest", str(manifest)]) + return rc, manifest + + +def test_manifest_lists_every_module_once_in_the_slot_shape(project, tmp_path): + rc, manifest = emit(project, tmp_path) + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["format"] == "typedpython-emit-c" and m["version"] == 1 + assert m["projectRoot"] == project.resolve().as_posix() + runtime = Path(m["runtimeInclude"]) + assert runtime.is_absolute() and (runtime / "tp_runtime.h").is_file() + assert m["compilerVersion"] + + native = {n["moduleName"]: n for n in m["nativeModules"]} + assert list(native) == ["app.fast"] + fast = native["app.fast"] + assert set(fast) >= {"moduleName", "sources", "includeDirs", "defines", "file", "status", "skippedItems"} + assert fast["file"] == "app/fast.py" and fast["defines"] == {} + assert fast["includeDirs"] == [m["runtimeInclude"]] + assert len(fast["sources"]) == 1 + c_file = Path(fast["sources"][0]) + assert c_file.is_absolute() and c_file.name == "fast.c" and c_file.is_file() + assert "PyInit_fast" in c_file.read_text() + + other = {n["module"]: n for n in m["notNativeModules"]} + assert list(other) == ["app.plain", "app.text"] + assert other["app.plain"] == {"module": "app.plain", "file": "app/plain.py", "kind": "NOT_MARKED", "items": []} + text = other["app.text"] + assert text["kind"] == "NOTHING_COMPILED" and text["items"] + item = text["items"][0] + assert set(item) == {"name", "file", "line", "message"} + assert item["name"] == "f" and item["file"] == "app/text.py" and isinstance(item["line"], int) + + every = [n["moduleName"] for n in m["nativeModules"]] + [n["module"] for n in m["notNativeModules"]] + assert "app" not in every and "app.__init__" not in every # a package __init__ is in neither + + +def test_a_second_run_reuses_the_same_c_files(project, tmp_path): + _, manifest = emit(project, tmp_path) + first = json.loads(manifest.read_text()) + c_file = Path(first["nativeModules"][0]["sources"][0]) + before = c_file.read_bytes() + assert first["nativeModules"][0]["status"] == "rebuilt" + rc, manifest = emit(project, tmp_path) + second = json.loads(manifest.read_text()) + assert rc == 0 + assert second["nativeModules"][0]["status"] == "reused" + assert second["nativeModules"][0]["sources"] == first["nativeModules"][0]["sources"] + assert c_file.read_bytes() == before + strip = lambda d: {**d, "nativeModules": [{**n, "status": None} for n in d["nativeModules"]]} + assert strip(second) == strip(first) + + +def test_files_given_one_by_one_keep_their_order(project, tmp_path): + rc, manifest = emit(project, tmp_path, project / "app" / "text.py", project / "app" / "plain.py") + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["nativeModules"] == [] + assert [n["module"] for n in m["notNativeModules"]] == ["app.text", "app.plain"] + + +def test_a_gate_error_exits_1_and_writes_no_manifest(project, tmp_path, capsys): + (project / "app" / "broken.py").write_text(BROKEN) + rc, manifest = emit(project, tmp_path) + assert rc == 1 + assert not manifest.exists() + assert "typedpython:" in capsys.readouterr().err + + +def test_no_python_files_exits_2(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + rc, manifest = emit(empty, tmp_path) + assert rc == 2 and not manifest.exists() diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index 51f1bca43..d02182c6d 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -2,9 +2,10 @@ signals reach Python (KeyboardInterrupt), and other threads get the GIL. A Python signal handler, a pending call and another thread are user code that can change lists and -globals. A function that holds an array copy-in (ArrayParam) or an entry-globals snapshot runs no -user code by contract, so while such a function is active the poll is deferred: the result is then -the CPython execution in which the signal arrived after the call. +globals. A function that holds an array copy-in (ArrayParam) or an entry-globals snapshot polls +too (#152, #335, SPEC N-8): at a poll it writes its lists back, runs the handler, copies them in +again and re-checks its entry guards, so the result includes what the handler wrote (or, after a +poll it cannot continue from, it leaves for the interpreted continuation). Each scenario runs in a child interpreter, so a broken build cannot take pytest down and signal handlers stay out of the test process. Timing thresholds are generous; they separate "polled" @@ -18,14 +19,15 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def spin(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -33,12 +35,14 @@ def spin(n: int) -> float: return acc +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -47,7 +51,7 @@ def total(xs: list[float], rounds: int) -> float: return s """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, signal, sys, threading, time path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("loops", path) @@ -133,12 +137,17 @@ def worker(): @posix_signals -def test_a_handler_is_deferred_while_an_array_copy_is_live(extension): +def test_a_handler_during_an_array_copy_is_seen_by_the_result(extension): # total() reads its list through a copy-in. A handler that writes the list mid-call would be - # invisible to the copy, a result no CPython execution can give. So the handler must run after - # the call (the execution where the signal arrived late), or the result must include its write. + # invisible to a stale copy, a result no CPython execution can give. So either the handler runs + # after the call (the execution where the signal arrived late) or the result includes its write + # (a poll wrote the list back, ran the handler and copied it in again, #152/#335). + # Size (#406): each poll inside the call re-copies the whole list, so the list stays under + # TP_POLL_INTERVAL (4096) and the inner loop never polls; with 100_000 elements a call made + # 480_000 full re-copies and timed out at 120 s on slow macOS runners. The outer loop still polls + # (once per 4096 rounds); the inner-loop poll and the OSR exit are covered by test_osr_exit.py. r = run_child(extension, """ - xs = [1.0] * 100_000 + xs = [1.0] * 4_000 seen = {} def handler(signum, frame): seen["t"] = time.perf_counter() @@ -147,10 +156,10 @@ def handler(signum, frame): import os threading.Timer(0.05, lambda: os.kill(os.getpid(), signal.SIGUSR1)).start() t0 = time.perf_counter() - s = m.total(xs, 20_000) + s = m.total(xs, 500_000) # 2e9 additions, as before t1 = time.perf_counter() time.sleep(0.01) # let a deferred handler run - print(json.dumps({"s": s, "plain": 100_000 * 20_000 * 1.0, "t0": t0, "t1": t1, + print(json.dumps({"s": s, "plain": 4_000 * 500_000 * 1.0, "t0": t0, "t1": t1, "handler": seen.get("t")})) """) assert r["handler"] is not None, "the handler never ran" @@ -165,9 +174,9 @@ def handler(signum, frame): # ---- chunked polling for range loops (#206, SPEC N-8) --------------------------------------------- CHUNK_SOURCE = """\ -# typedpython: compiled +@compiled def brk(n: int, k: int) -> int: s: int = 0 last: int = -1 @@ -179,6 +188,7 @@ def brk(n: int, k: int) -> int: return s * 3 + last +@compiled def cont(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -188,6 +198,7 @@ def cont(n: int, m: int) -> int: return s +@compiled def ret(n: int, k: int) -> int: for i in range(n): if i == k: @@ -195,6 +206,7 @@ def ret(n: int, k: int) -> int: return -1 +@compiled def down(n: int) -> int: s: int = 0 for i in range(n, 0, -3): @@ -202,6 +214,7 @@ def down(n: int) -> int: return s +@compiled def down1(n: int) -> int: s: int = 0 for i in range(n, -1, -1): @@ -209,6 +222,7 @@ def down1(n: int) -> int: return s +@compiled def up3(a: int, b: int) -> int: s: int = 0 for i in range(a, b, 3): @@ -216,6 +230,7 @@ def up3(a: int, b: int) -> int: return s +@compiled def nested(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -226,6 +241,7 @@ def nested(n: int, m: int) -> int: return s +@compiled def brk_in_while(n: int) -> int: s: int = 0 for i in range(n): @@ -238,6 +254,7 @@ def brk_in_while(n: int) -> int: return s +@compiled def huge(a: int, b: int, cap: int) -> int: c: int = 0 for i in range(a, b): @@ -247,6 +264,7 @@ def huge(a: int, b: int, cap: int) -> int: return c +@compiled def huge_last(a: int, b: int, cap: int) -> int: c: int = 0 last: int = 0 @@ -346,7 +364,7 @@ def test_range_loop_polls_once_per_chunk(chunk_module): def test_while_loop_keeps_its_per_iteration_countdown(tmp_path): path = tmp_path / "w.py" - path.write_text("# typedpython: compiled\n\n\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" + path.write_text("\n\n@compiled\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" " i += 1\n return i\n") body = _function_c(_generated_c(path), "w") lines = body.splitlines() @@ -379,13 +397,16 @@ def test_keyboard_interrupt_reaches_a_long_range_loop_in_chunks(extension): MID_SOURCE = """\ -# typedpython: compiled +@compiled def mid(n: int) -> float: x: float = 1.0 for k in range(n): - x = x * 0.9999999 + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 2.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 3.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 4.0) return x """ @@ -407,7 +428,7 @@ def handler(signum, frame): t0 = time.perf_counter() m.mid(n) full = time.perf_counter() - t0 - signal.setitimer(signal.ITIMER_REAL, full / 10) + signal.setitimer(signal.ITIMER_REAL, full / 20) t0 = time.perf_counter() try: m.mid(n) @@ -418,4 +439,5 @@ def handler(signum, frame): print(json.dumps({"out": out, "full": full, "elapsed": time.perf_counter() - t0})) """) assert r["out"] == "interrupted", r + assert r["full"] > 0.1, r assert r["elapsed"] < r["full"] * 0.8, r # before #280 the signal waited for the whole loop diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index f1e43a1ac..633a14487 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -20,50 +20,59 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def helper(x: int) -> float: d: dict[str, int] = {} d['a'] = x return d['a'] * 2.5 +@compiled def leaf(i: int) -> float: return 1.0 / (i + 1) +@compiled def mid(i: int) -> float: return leaf(i) + leaf(i + 1) +@compiled def top(i: int) -> float: return mid(i) * 2.0 +@compiled def objy(i: int) -> float: return float(helper(i)) + leaf(i) +@compiled def viaobj(i: int) -> float: return objy(i) + 1.0 +@compiled def rec(n: int) -> int: if n == 0: return 0 return 1 + rec(n - 1) +@compiled def callrec(n: int) -> int: return rec(n) + 1 +@compiled def loopy(n: int) -> float: s: float = 0.0 for i in range(n): @@ -71,14 +80,17 @@ def loopy(n: int) -> float: return s +@compiled def outer(n: int) -> float: return loopy(n) + 1.0 +@compiled def cube(x: int) -> float: return float(x * x * x) + leaf(x) +@compiled def fill(x: int, out: list[float]) -> None: out[0] = cube(x) """ @@ -159,7 +171,7 @@ def test_the_fast_impl_is_never_addressed(traced): assert "&tp_fimpl_" not in c and "(tp_fimpl_" not in c -LOAD = """\ +LOAD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys def load(name, path, ext): if ext: diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index 378395e78..88db26a2b 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -4,7 +4,8 @@ * declare `Py_mod_gil = Py_MOD_GIL_NOT_USED`, so importing it does not switch the GIL back on; * keep its shared mutable state thread-safe (runtime/API.md "Free-threaded CPython"); * read nothing through a borrowed reference that another thread can free; - * hold the list's critical section from before the copy-in to after the write-back (array params). + * hold the list's critical section from before the copy-in to after the write-back (array params); + a pure function with three or more list parameters reads them element by element instead (#255). Every concurrency scenario runs in a CHILD interpreter with a timeout (a crash or a deadlock fails one test, not the run). Iteration counts scale with TP_FT_STRESS (default 1). @@ -12,6 +13,11 @@ Red record (before the change, on the 3.15t venv): the GIL test and the three module-level checks failed (`Py_mod_gil` absent, the GIL came back at import), the concurrent counters lost updates, and a three-list function was compiled. + +#255 (three or more lists, design/typedpython.md 4.3.6): red before the change, the pure functions with +three and seven list parameters (`three`, `seven`, `order3`, `sum3`) were not compiled, so `built` failed +and every test of this file using it errored; the front-end, verifier and cgen tests of the 3-list +section failed (verify.ARRAY_DIRECT did not exist). """ from __future__ import annotations @@ -27,15 +33,16 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, pipeline, verify STRESS = max(1, int(os.environ.get("TP_FT_STRESS", "1"))) FREE_THREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) SOURCE = '''\ -# typedpython: compiled +@compiled class Cell: __slots__ = ("a", "b") a: object @@ -46,6 +53,7 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled class Twin: __slots__ = ("a", "b") a: object @@ -56,18 +64,22 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def geta(c: Cell) -> object: return c.a +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def mk(a: object, b: object) -> Cell: return Cell(a, b) +@compiled def fresh(a: object, n: int) -> object: c = Cell(a, None) keep: object = c # read as a value: c is a real object, not virtual (N-12a) @@ -77,11 +89,13 @@ def fresh(a: object, n: int) -> object: return s +@compiled def vpick(a: object, b: object) -> object: c = Cell(a, b) return c.b +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -90,24 +104,29 @@ def total(xs: list[float], rounds: int) -> float: return s +@compiled def fill(xs: list[float], v: float) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge(xs: list[float]) -> float: return xs[0] - xs[len(xs) - 1] +@compiled def fill_int(xs: list[int], v: int) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge_int(xs: list[int]) -> int: return xs[0] - xs[len(xs) - 1] +@compiled def rotate(xs: list[float]) -> None: if len(xs) > 1: first: float = xs[0] @@ -116,6 +135,7 @@ def rotate(xs: list[float]) -> None: xs[len(xs) - 1] = first +@compiled def zero2(xs: list[int], ys: list[int]) -> int: for i in range(len(xs)): xs[i] = 0 @@ -124,22 +144,81 @@ def zero2(xs: list[int], ys: list[int]) -> int: return 0 +@compiled def pair_edge(xs: list[int], ys: list[int]) -> int: return xs[0] - ys[len(ys) - 1] +@compiled def three(a: list[int], b: list[int], c: list[int]) -> int: return len(a) + len(b) + len(c) +@compiled +def bump1(a: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + + +@compiled +def bump2(a: list[float], b: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + for i in range(len(b)): + b[i] = b[i] + 1.0 + + +@compiled +def bump3(a: list[float], b: list[float], c: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + for i in range(len(b)): + b[i] = b[i] + 1.0 + for i in range(len(c)): + c[i] = c[i] + 1.0 + + +@compiled +def seven(x: list[float], y: list[float], z: list[float], vx: list[float], vy: list[float], + vz: list[float], w: list[float]) -> float: + e: float = 0.0 + for i in range(len(x)): + e += 0.5 * w[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, len(x)): + dx: float = x[i] - x[j] + dy: float = y[i] - y[j] + dz: float = z[i] - z[j] + e -= w[i] * w[j] * (dx * dx + dy * dy + dz * dz) + return e + + +@compiled +def order3(a: list[float], b: list[float], c: list[float]) -> float: + x: float = a[0] + y: float = b[0] + z: float = c[0] + return z - x + 0.0 * y + + +@compiled +def sum3(a: list[float], b: list[float], c: list[float]) -> float: + return a[0] + b[0] + c[0] + + K: int = 3 +@compiled +def three_k(a: list[int], b: list[int], c: list[int]) -> int: + return len(a) + len(b) + len(c) + K + + +@compiled def usek(x: int) -> int: return x + K ''' -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import faulthandler, importlib.machinery, importlib.util, json, sys, threading, time faulthandler.enable() path = sys.argv[1] @@ -179,7 +258,7 @@ def built(tmp_path_factory): assert result.extension is not None, result.skipped compiled = {f.name for f in result.module.functions} wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2", - "pair_edge", "usek", "fresh", "vpick"} + "pair_edge", "usek", "fresh", "vpick", "three", "bump1", "bump2", "seven", "order3", "sum3"} assert wanted <= compiled, {n: result.skipped.get(n) for n in wanted - compiled} fns = {f.name: f for f in result.module.functions} assert fns["vpick"].virtual == ("c",) and fns["fresh"].virtual == () # #143: one virtual, one real @@ -233,27 +312,70 @@ def test_no_runtime_warning_about_the_gil(extension): assert "global interpreter lock" not in proc.stderr -# --- more than two list parameters stay interpreted -------------------------------------------------- +# --- three or more list parameters (#255) ----------------------------------------------------------- +# CPython's critical sections lock one or two objects, and the leader decision of 2026-11-13 excludes any +# other lock (design/typedpython.md 4.3.6). A function with an effect and more than two lists stays +# interpreted, as since #209. A pure one (ir.direct_lists) reads its lists element by element with +# PyList_GetItemRef, in program order, without copy-in or lock; an element of another type deopts. + +def test_pure_three_and_seven_list_functions_compile(built): + result, _, _ = built + assert {"three", "seven", "order3", "sum3"} <= {f.name for f in result.module.functions}, \ + result.skipped + -def test_three_list_parameters_stay_interpreted(built): +def test_three_list_parameters_with_an_effect_stay_interpreted(built): result, _, _ = built - assert "three" not in {f.name for f in result.module.functions} - reason = result.skipped["three"] - assert "list" in reason and ("three" in reason or "more than two" in reason), reason + assert "bump3" not in {f.name for f in result.module.functions} + reason = result.skipped["bump3"] + assert "more than two list parameters" in reason, reason def test_front_end_gives_a_clear_reason(tmp_path): p = tmp_path / "m.py" - p.write_text("# typedpython: compiled\n\n\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" - " return len(a)\n") + p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> None:\n" + " a[0] = 1\n") m = frontend.lower(p) assert not m.functions assert "more than two list parameters" in m.skipped["f"], m.skipped["f"] +def test_front_end_lowers_three_list_parameters_of_a_pure_function(tmp_path): + p = tmp_path / "m.py" + p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" + " return len(a)\n") + m = frontend.lower(p) + assert [(f.name, f.pure, f.may_deopt) for f in m.functions] == [("f", True, True)], m.skipped + + +def test_three_lists_with_an_entry_global_stay_interpreted(built): + """A module global is read once at entry; the lists are read later, one element at a time, so the + two reads could disagree with any interpreted order.""" + result, _, _ = built + assert "three_k" not in {f.name for f in result.module.functions} + reason = result.skipped["three_k"] + assert "more than two list parameters" in reason and "global" in reason, reason + + def test_two_list_parameters_still_compile(built): result, _, _ = built - assert {"zero2", "pair_edge"} <= {f.name for f in result.module.functions} + assert {"zero2", "pair_edge", "bump2"} <= {f.name for f in result.module.functions} + + +def _wrapper(c_text: str, name: str) -> str: + start = c_text.index(f"static PyObject *tp_wrap_{name}(") + return c_text[start:c_text.index("\n}\n", start)] + + +def test_direct_lists_take_no_lock_and_are_not_copied(built): + result, path, _ = built + c_text = cgen.generate(result.module, path) + assert "tp_cs_begin(&tp_lock" in _wrapper(c_text, "zero2") # two lists: unchanged + for name in ("three", "seven", "order3"): + w = _wrapper(c_text, name) + assert "tp_cs_" not in w and "_array_enter" not in w and "_array_exit" not in w, w + assert "PyList_CheckExact" in w, w + assert "tp_list_ld_f64(l_a->list" in c_text def test_verifier_rejects_three_array_params(built): @@ -279,6 +401,70 @@ def test_cgen_refuses_three_array_params(built): cgen.generate(mod, path) +def _three_with_entry_global(result): + from typedpython import ir + f = next(f for f in result.module.functions if f.name == "three") + bad = dataclasses.replace(f, entry_globals=(ir.Param("K", ir.Type.I64),)) + return dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) + + +def test_verifier_rejects_direct_lists_with_an_entry_global(built): + result, _, _ = built + proved, diags = verify.verify(_three_with_entry_global(result)) + assert not proved.functions + assert any(d.rule == verify.ARRAY_DIRECT and "entry global" in d.message for d in diags), diags + + +def test_cgen_refuses_direct_lists_with_an_entry_global(built): + result, path, _ = built + with pytest.raises(cgen.CGenError, match="entry global"): + cgen.generate(_three_with_entry_global(result), path) + + +def test_three_and_seven_lists_match_the_interpreter(extension): + r = run_child(extension, """ + import random + rng = random.Random(255) + interp = m.__typedpython_interpreted__ + def both(name, *args): + out = [] + for fn in (getattr(m, name), interp[name]): + try: + out.append(("ok", fn(*[list(a) if type(a) is list else a for a in args]))) + except Exception as e: + out.append((type(e).__name__, str(e))) + return out + before = m.__typedpython_deopts__ + bad = [] + for n in (0, 1, 2, 5, 40): + ls = [[rng.uniform(-3.0, 3.0) for _ in range(n)] for _ in range(7)] + for name, args in (("seven", ls), ("three", [list(range(n)), list(range(n + 1)), []]), + ("order3", ls[:3]), ("sum3", ls[:3])): + c, i = both(name, *args) + if c != i: + bad.append((name, n, c, i)) + same_list = [1.0, 2.0] + if m.order3(same_list, same_list, same_list) != interp["order3"](same_list, same_list, same_list): + bad.append("alias") + compiled_deopts = m.__typedpython_deopts__ - before + # deopts (then interpreted): an element of another type met by a read, a non-list argument + ls = [[1.0, 2.0, 3.0] for _ in range(7)] + ls[6] = [1.0, 2, 3.0] + for name, args in (("seven", ls), ("three", [[1], (2, 3), [4]])): + c, i = both(name, *args) + if c != i: + bad.append((name, "deopt", c, i)) + # an IndexError where the interpreter raises it (a shorter list), not a deopt + for args in ([[1.0], [], [2.0]], [[], [1.0], [2.0]]): + c, i = both("sum3", *args) + if c != i or c[0] != "IndexError": + bad.append(("sum3", "index", c, i)) + print(json.dumps({"bad": repr(bad), "deopts": compiled_deopts, + "deopted": m.__typedpython_deopts__ - before - compiled_deopts})) + """) + assert r == {"bad": "[]", "deopts": 0, "deopted": 2}, r + + # --- concurrent calls -------------------------------------------------------------------------------- def test_private_lists_in_many_threads(extension): @@ -418,6 +604,166 @@ def look(): assert r == {"bad": [], "errs": []} +# Three shared lists read element by element (#255). The writer stores c[0] and then a[0], each one +# atomic store, with a growing value; order3 reads a[0] and then c[0]. The interpreter (and every +# compiled execution equal to an interleaving of it) sees c[0] >= a[0]: having seen the writer's a[0] +# it must see its earlier c[0]. A read out of program order (a copy of c taken before the read of a, +# as a per-list copy-in would do) can see a new a[0] with an old c[0]: the mutant below. +ORDER_BODY = """ + a = [0.0] + b = [0.0] + c = [0.0] + stop = False + N = WRITES + def writer(): + global stop + for k in range(1, N): + c[0] = float(k) + a[0] = float(k) + stop = True + bad = [] + calls = [0] + def reader(): + while True: + done = stop + d = m.order3(a, b, c) + calls[0] += 1 + if d < 0.0: + bad.append(d) + if done: + break + errs = run_threads([writer, reader, reader, reader]) + print(json.dumps({"bad": len(bad), "calls": calls[0], "errs": errs})) +""" + + +def test_three_lists_are_read_in_program_order(extension): + r = run_child(extension, ORDER_BODY.replace("WRITES", "(200000 * STRESS)")) + assert r["bad"] == 0 and r["errs"] == [] and r["calls"] > 0, r + + +def _read_order_mutant(built, tmp: Path) -> Path: + """order3 with its reads of a[0] and c[0] swapped (c first), in its impl and its fast twin.""" + result, path, _ = built + c_text = cgen.generate(result.module, path) + ra = " tp_s = tp_list_ld_f64(l_a->list, INT64_C(0), &tp_t0);\n" + rc = " tp_s = tp_list_ld_f64(l_c->list, INT64_C(0), &tp_t2);\n" + out, pos, done = [], 0, 0 + for head in re.finditer(r"^static (?:inline )?int tp_f?impl_order3\([^\n]*\)\n\{\n", c_text, re.M): + end = c_text.index("\n}\n", head.end()) + body = c_text[head.end():end] + assert body.count(ra) == 1 and body.count(rc) == 1 and body.count(" l_x = tp_t0;\n") == 1, body + body = (body.replace(ra, "@A@").replace(rc, ra).replace("@A@", rc) # c read first, a last + .replace(" l_x = tp_t0;\n", "").replace(" l_z = tp_t2;\n", "") + .replace(" tp_t3 = l_z - l_x;\n", " l_z = tp_t2;\n l_x = tp_t0;\n tp_t3 = l_z - l_x;\n")) + assert " l_x = tp_t0;\n" in body, body + out += [c_text[pos:head.end()], body] + pos = end + done += 1 + assert done >= 1, "order3's impl not found in the generated C" + out.append(c_text[pos:]) + return cbuild.build("".join(out), "ftmod", tmp / "read_order", ext_suffix=None) + + +@pytest.mark.race_mutant +def test_mutant_reading_out_of_program_order_is_caught(built, tmp_path): + so = _read_order_mutant(built, tmp_path) + caught = False + for _ in range(3): + r = run_child(so, ORDER_BODY.replace("WRITES", "400000")) + if r["bad"]: + caught = True + break + assert caught, "the mutant (c[0] read before a[0]) was not caught" + + +def test_element_of_another_type_deopts_under_a_concurrent_writer(extension): + """b[0] flips between 1 and 1.0 while sum3 reads it: every result is 3.0 (the interpreter's, after + a deopt when the read meets the int), never a float read out of an int.""" + r = run_child(extension, """ + a, b, c = [1.0], [1], [1.0] + stop = False + def writer(): + global stop + for k in range(100000 * STRESS): + b[0] = 1 if k % 2 else 1.0 + stop = True + bad = [] + before = m.__typedpython_deopts__ + def reader(): + while True: + done = stop + v = m.sum3(a, b, c) + if v != 3.0: + bad.append(v) + if done: + break + errs = run_threads([writer, reader, reader]) + print(json.dumps({"bad": bad[:3], "errs": errs, "deopted": m.__typedpython_deopts__ > before})) + """) + assert r == {"bad": [], "errs": [], "deopted": True}, r + + +def test_mutant_without_the_element_type_check_is_caught(built, tmp_path): + """The weakest point of the element-by-element read: tp_list_ld_f64's exact-float check. Without + it an int element is read as a double (deterministic, no race needed).""" + result, path, _ = built + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + check = (" if (!PyFloat_CheckExact(v)) {\n Py_DECREF(v);\n return 1;\n }\n" + " *out = PyFloat_AS_DOUBLE(v);\n") + assert h.count(check) == 1 + d = tmp_path / "rt" + d.mkdir() + (d / "tp_runtime.h").write_text(h.replace(check, " *out = PyFloat_AS_DOUBLE(v);\n")) + so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=d) + body = """ + print(json.dumps({"v": m.sum3([1.0], [1], [1.0])})) + """ + assert run_child(result.extension, body) == {"v": 3.0} + try: + caught = run_child(so, body, timeout=60)["v"] != 3.0 + except (AssertionError, subprocess.TimeoutExpired): + caught = True + assert caught, "the mutant (no exact-float check) was not caught" + + +def test_caller_held_critical_section(extension): + """list.sort holds the list's critical section while it calls the key (listobject.c, @critical_section + on list_sort_impl, and it empties the list meanwhile). A key that calls a three-list function with + that list, with a list of an outer sort, or with other lists, must not wait for its own thread, and + every call returns or raises what the interpreted function does in the same situation.""" + r = run_child(extension, """ + interp = m.__typedpython_interpreted__ + def run(fn): + log = [] + def call(*args): + try: + log.append(("ok", fn(*args))) + except Exception as e: + log.append((type(e).__name__, str(e))) + a = [3.0, 1.0, 2.0] + b = [5.0] * 4 + c = [7.0] * 4 + a.sort(key=lambda v: (call(a, b, c), call(b, a, c), call(b, c, b), v)[3]) + outer = [2.0, 1.0] + inner = [5.0, 4.0, 6.0] + def k_inner(w): + call(outer, b, inner) + call(b, c, b) + return w + def k_outer(v): + inner.sort(key=k_inner) + call(c, b, c) + return v + outer.sort(key=k_outer) + return log, a, b, c, outer, inner + got = run(m.order3) + want = run(interp["order3"]) + print(json.dumps({"same": got == want, "got": repr(got)})) + """, timeout=120) + assert r["same"], r["got"] + + def test_shared_objects_concurrent_fieldset_and_fieldget(extension): r = run_child(extension, """ cell = m.Cell((0,), None) @@ -593,8 +939,45 @@ def reader(): assert detected, "the mutant (no critical section around copy-in) was not caught" -def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool) -> Path: - """A copy of tp_runtime.h with one #159 protection removed (the mutation of that protection).""" +# Mutant-build-only scheduling helpers (#399). A mutant of the real header may contain these; the real +# header never does. TP_MUT_YIELD gives up the CPU (POSIX sched_yield, Windows SwitchToThread) so the +# thread a mutant waits for can run even when the runner has fewer vCPUs than runnable threads. +# TP_MUT_SPIN_MAX bounds every wait, so a mutant never hangs: it only loses the widened window. +_MUT_PRELUDE = """#include +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#define TP_MUT_YIELD() ((void)SwitchToThread()) +#else +#include +#define TP_MUT_YIELD() ((void)sched_yield()) +#endif +#define TP_MUT_SPIN_MAX 4000000L +""" + + +def _with_mut_prelude(h: str) -> str: + assert h.count("#include \n") == 1 + return h.replace("#include \n", _MUT_PRELUDE, 1) + + +def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool, hold_watch_open: bool = False) -> Path: + """A copy of tp_runtime.h with one #159 protection removed (the mutation of that protection). + + hold_watch_open (the dict-lock mutant only) makes the window deterministic. The real code is + sound because the watcher (which moves tp_globals_epoch) and the store of the new binding both + run under the dict's lock, which tp_class_global_ok also takes. Without that lock the race needs + bump < reader's epoch load < reader's lookup < store, and the writer's bump-to-store gap is a few + tens of nanoseconds: a reader gets into it only by luck, and on a 3-4 vCPU runner with seven + runnable threads it rarely does. So the mutant build itself holds the gap open: right after the + watcher moves the epoch it waits (bounded, yielding) until another thread has looked the old + binding up under the new epoch AND remembered it in rt->gepoch. Only then does the store of the + new binding go ahead. The remembered binding is then stale by construction.""" h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() g = h if drop_dict_lock: @@ -606,6 +989,22 @@ def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool) -> Path g = g.replace("tp_atomic_load_u64(&tp_globals_epoch)", "tp_globals_epoch") g = g.replace("tp_atomic_load_u64(&rt->gepoch)", "rt->gepoch") g = g.replace("tp_atomic_store_u64(&rt->gepoch, epoch)", "rt->gepoch = epoch") + if hold_watch_open: + assert drop_dict_lock and not plain_epoch + g = _with_mut_prelude(g) + decl = "static uint64_t tp_globals_epoch = 1;\n" + assert g.count(decl) == 1 + g = g.replace(decl, decl + "static uint64_t tp_mut_looked = 0; /* mutant only: last epoch a reader remembered */\n" + "#define TP_MUT_HOLD_OPEN() do { uint64_t want_ = tp_atomic_load_u64(&tp_globals_epoch); long n_; \\\n" + " for (n_ = 0; n_ < TP_MUT_SPIN_MAX && tp_atomic_load_u64(&tp_mut_looked) < want_; n_++) { \\\n" + " if ((n_ & 63) == 63) { TP_MUT_YIELD(); } } } while (0)\n") + bump = "tp_atomic_add_u64(&tp_globals_epoch, 1);\n" + assert g.count(bump) == 2 + g = g.replace(bump, "tp_atomic_add_u64(&tp_globals_epoch, 1);\n TP_MUT_HOLD_OPEN();\n") + remember = " tp_atomic_store_u64(&rt->gepoch, epoch);\n }\n return rc;" + assert g.count(remember) == 1 + g = g.replace(remember, " tp_atomic_store_u64(&rt->gepoch, epoch);\n" + " tp_atomic_store_u64(&tp_mut_looked, epoch);\n }\n return rc;") assert g != h d = tmp / "rt" d.mkdir(parents=True, exist_ok=True) @@ -618,7 +1017,7 @@ def test_mutant_without_the_dict_critical_section_is_caught(built, tmp_path): """Without the critical section around the epoch read and the global lookup, a thread can remember a binding the rebinding thread has already replaced (the watcher fires before the store).""" result, path, _ = built - rt = _racy_runtime(tmp_path, drop_dict_lock=True, plain_epoch=False) + rt = _racy_runtime(tmp_path, drop_dict_lock=True, plain_epoch=False, hold_watch_open=True) so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=rt) body = """ stop = False @@ -633,6 +1032,7 @@ def hammer(): m.Cell = m.Twin if i % 2 == 0 else orig if type(m.mk(1, 2)).__name__ != ("Twin" if i % 2 == 0 else "Cell"): bad.append(i) + break stop = True ht.join() print(json.dumps({"stale": len(bad)})) @@ -654,15 +1054,21 @@ def test_mutant_with_a_borrowed_field_read_is_caught(built, tmp_path): h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() good = " return PyMember_GetOne((const char *)obj, m);\n" assert good in h + # The window is held open until the writer has replaced the slot (bounded, yielding), so the old + # value has lost the slot's reference before the borrowed pointer is turned into a reference: the + # mutant always resurrects a dead object (and every later release of it is a second dealloc), + # instead of relying on a writer that happens to run inside a fixed-length spin. racy = (" PyObject *v = *(PyObject *volatile *)((char *)obj + m->offset);\n" - " for (volatile int i = 0; i < 2000; i++) { }\n" + " { long n_; for (n_ = 0; n_ < TP_MUT_SPIN_MAX\n" + " && *(PyObject *volatile *)((char *)obj + m->offset) == v; n_++) {\n" + " if ((n_ & 63) == 63) { TP_MUT_YIELD(); } } }\n" " return Py_NewRef(v);\n") d = tmp_path / "rt" d.mkdir() - (d / "tp_runtime.h").write_text(h.replace(good, racy)) + (d / "tp_runtime.h").write_text(_with_mut_prelude(h.replace(good, racy))) so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=d) body = """ - cell = m.Cell((0, 0), None) + cell = m.Cell((0, [0]), None) stop = False bad = [] def writer(): @@ -673,8 +1079,10 @@ def writer(): def reader(): for _ in range(20000): v = m.geta(cell) - if type(v) is not tuple or len(v) != 2: + if (type(v) is not tuple or len(v) != 2 or type(v[1]) is not list + or len(v[1]) != 1 or v[1][0] != v[0]): bad.append(1) + break wt = threading.Thread(target=writer); wt.start() errs = run_threads([reader] * 3) stop = True @@ -823,12 +1231,14 @@ def _access_is_ours(block: str) -> bool: A report where one access is mimalloc initialising a fresh block (`_mi_malloc*` memset) is not ours either: the extension's earlier read hit memory a thread later freed (an exited thread's `PyThreadState`, read by `tp_traced`), and the free and the reuse both happen inside the - uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI).""" + uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI). The same + holds when the reused block is filled by libpython's `new_threadstate` (a memcpy of the template + thread state into the block an exited thread's state was freed from, seen on Linux CI).""" firsts = [] for part in block.split("\n\n")[:2]: frames = [ln for ln in part.splitlines() if ln.strip().startswith("#")] firsts.append(next((f for f in frames if "libtsan" not in f and "sanitizer_common" not in f), "")) - if any(" _mi_malloc" in f or " _mi_page_malloc" in f for f in firsts): + if any(" _mi_malloc" in f or " _mi_page_malloc" in f or " new_threadstate " in f for f in firsts): return False return any("ftmod" in f for f in firsts) diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 8b2a55fd9..caaf73bc2 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -12,6 +12,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, @@ -54,7 +55,7 @@ def nodes(root, cls): def test_scalar_loop_lowers_to_for_range(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def total(n: int, x: float) -> float: acc: float = 0.0 for k in range(n): @@ -328,8 +329,8 @@ def scale(a: list[float], b: list[float], k: float) -> None: a[i] = b[i] * k """) f = function(m, "scale") - # two list parameters at most (#159: the lists are locked from copy-in to write-back; CPython - # has no three-object critical section). A third one is test_free_threaded.py's. + # two list parameters at most in a function with an effect (#159: the lists are locked from + # copy-in to write-back); three or more, and pure ones (#255), are test_free_threaded.py's. assert f.params == ( ArrayParam("a", Type.F64_ARRAY, True), ArrayParam("b", Type.F64_ARRAY, False), Param("k", F64), ) @@ -633,16 +634,18 @@ def f(x: float) -> object: assert Global(OBJ, "K") in nodes(f, Global) -def test_module_marker_compiles_every_module_level_def(tmp_path): +def test_compiled_marks_every_decorated_module_level_def(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def a(x: float) -> float: return x + @compiled def b(x: float) -> float: return -x + @compiled @staticmethod def c(x: float) -> float: return x @@ -651,7 +654,79 @@ def c(x: float) -> float: assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] -def test_without_marker_only_compiled_functions_are_lowered(tmp_path): +def test_compiled_parallel_option_and_former_comment_marker(tmp_path): + m = lower(tmp_path, """ + @compiled(parallel=True) + def p(x: float) -> float: + return x + + @compiled(parallel=False) + def s(x: float) -> float: + return -x + + # typedpython: parallel + @compiled + def c(x: float) -> float: + return x * 2.0 + + @compiled(unknown=True) + def bad(x: float) -> float: + return x + """) + assert [(f.name, f.parallel) for f in m.functions] == [("p", True), ("s", False), ("c", False)] + assert set(m.skipped) == {"bad"} and "parallel" in m.skipped["bad"] + + +def test_compiled_fastmath_option_and_validation(tmp_path): + m = lower(tmp_path, """ + @compiled(fastmath=True) + def fm(x: float) -> float: + return x + + @compiled(fastmath=False) + def exact(x: float) -> float: + return -x + + @compiled(parallel=True, fastmath=True) + def both(x: float) -> float: + return x * 2.0 + + @compiled + def plain(x: float) -> float: + return x + 1.0 + + @compiled(fastmath=1) + def bad_val(x: float) -> float: + return x + + @compiled(other=True) + def bad_kw(x: float) -> float: + return x + """) + assert [(f.name, f.parallel, f.fastmath) for f in m.functions] == [ + ("fm", False, True), + ("exact", False, False), + ("both", True, True), + ("plain", False, False), + ] + assert set(m.skipped) == {"bad_val", "bad_kw"} + assert "fastmath" in m.skipped["bad_val"] + assert "fastmath" in m.skipped["bad_kw"] and "parallel" in m.skipped["bad_kw"] + + +def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): + m = lower(tmp_path, """ + # typedpython: compiled + def a(x: float) -> float: + return x + + def b(x: float) -> float: + return -x + """) + assert m.functions == () and m.skipped == {} + + +def test_without_a_decorator_only_compiled_functions_are_lowered(tmp_path): m = lower(tmp_path, """ x: int = 1 # typedpython: compiled @@ -828,7 +903,7 @@ def f(n: int) -> int: def test_nbody_kernels_lower(tmp_path): path = tmp_path / "nbody.py" shutil.copy(BENCH / "nbody.py", path) - path.write_text("# typedpython: compiled\n" + path.read_text()) + path.write_text(decorate_compiled(path.read_text())) m = frontend.lower(path) for name in ("advance", "energy", "offset_momentum"): function(m, name) @@ -847,7 +922,7 @@ def test_nbody_kernels_lower(tmp_path): @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_spectral_norm_kernels_lower(tmp_path): path = tmp_path / "spectral_norm.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "spectral_norm.py").read_text()) + path.write_text(decorate_compiled((BENCH / "spectral_norm.py").read_text())) m = frontend.lower(path) eval_a = function(m, "eval_a") assert eval_a.pure and eval_a.may_deopt @@ -1048,7 +1123,7 @@ def f(a: int, b: int) -> {annotation}: @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_fannkuch_lowers(tmp_path): path = tmp_path / "fannkuch.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "fannkuch.py").read_text()) + path.write_text(decorate_compiled((BENCH / "fannkuch.py").read_text())) m = frontend.lower(path) assert "fannkuch" in [f.name for f in m.functions], m.skipped.get("fannkuch") f = function(m, "fannkuch") diff --git a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py index 33c0473cc..a2daaeed9 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py @@ -18,6 +18,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, Assign, Box, Call, CallObject, CheckExact, ClassDecl, Const, ExprStmt, FieldGet, FieldSet, @@ -28,8 +29,6 @@ BINARY_TREES = Path(__file__).resolve().parents[4] / "benchmarks/typedpython/py/binary_trees.py" SLOTS_LINE = ' __slots__ = ("left", "right")\n' -MARK = "# typedpython: compiled\n" - POINT = """ class P: __slots__ = ("x", "y") @@ -44,7 +43,7 @@ def __init__(self, x: float, y: float) -> None: def lower(tmp_path, source, name="mod.py"): path = tmp_path / name - path.write_text(MARK + textwrap.dedent(source).lstrip("\n")) + path.write_text(decorate_compiled(textwrap.dedent(source).lstrip("\n"))) return frontend.lower(path) @@ -573,7 +572,7 @@ def test_binary_trees_with_slots_lowers_make_and_check(tmp_path): source = BINARY_TREES.read_text() assert "class Node:\n" + SLOTS_LINE in source # the benchmark itself declares the slots path = tmp_path / "binary_trees.py" - path.write_text(MARK + source) + path.write_text(decorate_compiled(source)) m = frontend.lower(path) assert cls(m, "Node") == ClassDecl("Node", ("left", "right"), True) @@ -598,7 +597,7 @@ def test_binary_trees_without_slots_stays_object_code(tmp_path): path = tmp_path / "binary_trees.py" source = BINARY_TREES.read_text() assert SLOTS_LINE in source - path.write_text(MARK + source.replace(SLOTS_LINE, "", 1)) # the run.py no-slots baseline + path.write_text(decorate_compiled(source.replace(SLOTS_LINE, "", 1))) # the run.py no-slots baseline m = frontend.lower(path) assert m.classes == () assert "`__slots__`" in m.skipped["Node"] diff --git a/python-multiplatform-ksp/src/test/python/test_generate_project.py b/python-multiplatform-ksp/src/test/python/test_generate_project.py new file mode 100644 index 000000000..f5ae50a94 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_generate_project.py @@ -0,0 +1,111 @@ +"""generate_project: TypedPython caches source -> C, the compile slot builds C -> extension (#113). + +Real Pyrefly; the only C build here is the one that proves the emitted `.c` is the same program +compile_project builds. +""" +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild, frontend, pipeline + +B = decorate_compiled(textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """)) +A = decorate_compiled(textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "pkg").mkdir(parents=True) + (root / "a.py").write_text(A) + (root / "b.py").write_text(B) + (root / "pkg" / "__init__.py").write_text("") + (root / "pkg" / "plain.py").write_text(PLAIN) + return root + + +@pytest.fixture +def counts(monkeypatch): + n = {"lower": 0, "build": 0} + real_lower = frontend.lower + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + raise AssertionError("generate_project must not run the C compiler") + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def gen(project, cache): + return pipeline.generate_project( + [project / "a.py", project / "b.py", project / "pkg" / "plain.py"], project, cache) + + +def by_stem(result): + return {m.path.stem: m for m in result.modules} + + +def test_emits_one_c_per_compiled_module_and_the_runtime_dir(project, tmp_path, counts): + r = gen(project, tmp_path / "cache") + m = by_stem(r) + assert m["a"].c_source.name == "a.c" and m["b"].c_source.name == "b.c" + assert "PyInit_a" in m["a"].c_source.read_text() + assert m["a"].extension is None and m["a"].kind == "compiled" + assert m["plain"].c_source is None and m["plain"].kind == "not_marked" + assert (r.runtime_include / "tp_runtime.h").is_file() + assert counts == {"lower": 3, "build": 0} + + +def test_unchanged_module_hits_with_byte_identical_c(project, tmp_path, counts): + first = by_stem(gen(project, tmp_path / "cache")) + c_a = first["a"].c_source.read_bytes() + second = gen(project, tmp_path / "cache") + assert {m.path.stem: m.status for m in second.modules} == {"a": "reused", "b": "reused", "plain": "reused"} + assert by_stem(second)["a"].c_source.read_bytes() == c_a + assert counts["lower"] == 3 # nothing lowered again + + +def test_fresh_cache_gives_byte_identical_c(project, tmp_path): + one = by_stem(gen(project, tmp_path / "c1"))["a"].c_source.read_bytes() + two = by_stem(gen(project, tmp_path / "c2"))["a"].c_source.read_bytes() + assert one == two + + +def test_body_change_regenerates_only_that_module(project, tmp_path): + gen(project, tmp_path / "cache") + (project / "b.py").write_text(B.replace("n * 2", "n + n")) + st = {m.path.stem: m.status for m in gen(project, tmp_path / "cache").modules} + assert st == {"a": "reused", "b": "rebuilt", "plain": "reused"} + + +def test_key_ignores_toolchain(project, tmp_path, monkeypatch): + gen(project, tmp_path / "cache") + monkeypatch.setattr(cbuild, "DEFAULT_FLAGS", (*cbuild.DEFAULT_FLAGS, "-DDIFFERENT")) + monkeypatch.setattr(pipeline.sysconfig, "get_platform", lambda: "other-platform") + assert all(m.status == "reused" for m in gen(project, tmp_path / "cache").modules) + + +def test_emitted_c_is_what_compile_project_builds(project, tmp_path): + c_mod = by_stem(gen(project, tmp_path / "ccache"))["b"] + built = by_stem(pipeline.compile_project([project / "b.py"], project, tmp_path / "bcache"))["b"] + so = cbuild.build(c_mod.c_source.read_text(), "b", tmp_path / "slot") + assert cbuild.load("b", so).twice(21) == 42 + assert built.extension is not None and built.c_source is None diff --git a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py index 045a593e3..64260ddf7 100644 --- a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py +++ b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py @@ -330,39 +330,44 @@ def test_a_mod_result_can_feed_a_proof(): # --- 2. end to end --------------------------------------------------------------------------------- SOURCE = '''\ -# typedpython: compiled +@compiled def dec_if_positive(n: int) -> int: if n > 0: return n - 1 return -1 +@compiled def dec_after_guard(n: int) -> int: if n <= 0: return 0 return n - 1 +@compiled def inc_below_max(n: int) -> int: if n < 9223372036854775807: return n + 1 return 0 +@compiled def inc_below_100(n: int) -> int: if n < 100: return n + 1 return 0 +@compiled def inc_if_positive(n: int) -> int: if n > 0: return n + 1 return 0 +@compiled def reverse_gap(a: int, b: int) -> int: lo: int = a hi: int = b @@ -374,6 +379,7 @@ def reverse_gap(a: int, b: int) -> int: return hi - lo + steps +@compiled def killed_guard(n: int, a: list[int]) -> int: if n < 100: n = a[0] @@ -381,6 +387,7 @@ def killed_guard(n: int, a: list[int]) -> int: return 0 +@compiled def killed_in_loop(c: int, a: list[int]) -> int: n: int = a[0] t: int = 0 @@ -391,6 +398,7 @@ def killed_in_loop(c: int, a: list[int]) -> int: return t +@compiled def parity(n: int) -> int: k: int = 0 s: int = 0 @@ -401,11 +409,13 @@ def parity(n: int) -> int: return s +@compiled def const_local() -> int: x: int = 41 return x + 1 +@compiled def range_shifts(a: int, b: int) -> int: if a < 0: return 0 @@ -415,26 +425,31 @@ def range_shifts(a: int, b: int) -> int: return t +@compiled def guarded_shift(n: int) -> int: if n >= 0: return n // 16 + n % 16 return 0 +@compiled def negative_shift(n: int) -> int: return n // 4 + n % 8 +@compiled def odd_divisor(n: int) -> int: if n >= 0: return n // 3 + n % 6 return 0 +@compiled def mod_then_add(n: int) -> int: return n % 10 + 1 +@compiled def pair_sums(a: list[int]) -> int: best: int = 0 for i in range(len(a)): diff --git a/python-multiplatform-ksp/src/test/python/test_module_report.py b/python-multiplatform-ksp/src/test/python/test_module_report.py index 8625cf18d..86ead4b97 100644 --- a/python-multiplatform-ksp/src/test/python/test_module_report.py +++ b/python-multiplatform-ksp/src/test/python/test_module_report.py @@ -8,16 +8,17 @@ import textwrap from pathlib import Path +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, pipeline, verify -SRC = "# typedpython: compiled\n" + textwrap.dedent("""\ +SRC = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 def boom(s: str) -> str: raise ValueError("boom " + s) - """) + """)) def make_project(root: Path) -> Path: @@ -95,14 +96,14 @@ def test_a_moved_file_with_the_same_content_does_not_reuse_the_extension(tmp_pat # --- #128 ---------------------------------------------------------------------------------------- -MIXED = "# typedpython: compiled\n" + textwrap.dedent("""\ +MIXED = decorate_compiled(textwrap.dedent("""\ def ok(n: int) -> int: return n + 1 def greet(name: str) -> str: return name[1:] - """) + """)) def build_one(tmp_path, rel, source, cache="cache"): @@ -119,20 +120,20 @@ def test_kind_compiled(tmp_path): assert m.kind == "compiled" and m.extension is not None [item] = m.skipped_items assert item.name == "greet" and item.file == "pkg/mixed.py" - assert item.line == 7 and not item.message.startswith("line ") - assert m.skipped["greet"].startswith("line 7: ") # ModuleResult.skipped unchanged + assert item.line == 8 and not item.message.startswith("line ") + assert m.skipped["greet"].startswith("line 8: ") # ModuleResult.skipped unchanged def test_kind_nothing_compiled_with_structured_items(tmp_path): _, _, m = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") assert m.kind == "nothing_compiled" and m.extension is None [item] = m.skipped_items - assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 4) + assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 3) assert item.message and not re.match(r"line \d+", item.message) -def test_kind_not_marked_when_there_is_neither_marker_nor_decorator(tmp_path): +def test_kind_not_marked_when_there_is_no_decorator(tmp_path): _, _, m = build_one(tmp_path, "pkg/unmarked.py", "def f(x: int) -> int:\n return x\n") assert m.kind == "not_marked" and m.extension is None and m.skipped_items == [] @@ -145,15 +146,15 @@ def test_a_compiled_decorator_alone_opts_in(tmp_path): def test_class_items_carry_name_and_line(tmp_path): - src = ("# typedpython: compiled\n\n\nclass Node:\n __slots__ = ('a',)\n" + src = ("@compiled\nclass Node:\n __slots__ = ('a',)\n" " def __init__(self, a: int) -> None:\n self.a = a\n") _, _, m = build_one(tmp_path, "pkg/cls.py", src) cls = [i for i in m.skipped_items if i.name == "Node"] - assert cls and cls[0].line == 4 and cls[0].file == "pkg/cls.py" + assert cls and cls[0].line == 2 and cls[0].file == "pkg/cls.py" def test_verifier_skips_carry_a_line_and_no_prefix(tmp_path): - src = "# typedpython: compiled\n\n\ndef f(x: int) -> int:\n return g(x)\n\n\ndef g(x: int) -> int:\n return x\n" + src = "@compiled\ndef f(x: int) -> int:\n return g(x)\n\n\n@compiled\ndef g(x: int) -> int:\n return x\n" _, _, m = build_one(tmp_path, "pkg/v.py", src) for i in m.skipped_items: assert isinstance(i.line, int) and not i.message.startswith("line ") @@ -170,7 +171,7 @@ def test_structure_round_trips_through_the_cache_for_reused_modules(tmp_path): def test_not_marked_and_nothing_compiled_survive_reuse(tmp_path): root, path, _ = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") again = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] assert again.status == "reused" and again.kind == "nothing_compiled" and again.skipped_items root2, path2, _ = build_one(tmp_path, "pkg/un.py", "def f(x: int) -> int:\n return x\n") @@ -191,5 +192,5 @@ def test_cli_build_prints_file_line_name_message_and_not_marked(tmp_path): [sys.executable, "-m", "typedpython", "build", str(root), "--cache", str(tmp_path / "c")], capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(src))) assert out.returncode == 0, out.stdout + out.stderr - assert re.search(r"^\s*pkg/mixed\.py:7: greet: \S", out.stdout, re.M), out.stdout + assert re.search(r"^\s*pkg/mixed\.py:8: greet: \S", out.stdout, re.M), out.stdout assert re.search(r"pkg/un\.py.*not marked for compilation", out.stdout), out.stdout diff --git a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py index f2ceeb875..2da246f90 100644 --- a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py +++ b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py @@ -28,13 +28,13 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, cost, frontend, ir, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled import math as GMOD @@ -42,14 +42,17 @@ MOD: int = 2147483647 + @compiled def _one() -> int: return 1 + @compiled def _two() -> int: return 2 + @compiled class GTYPE: pass @@ -57,6 +60,7 @@ class GTYPE: GFN = _one + @compiled def add2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + b @@ -66,6 +70,7 @@ def add2(log: list[object], a: int, b: int) -> object: return r + @compiled def sub2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - b @@ -75,6 +80,7 @@ def sub2(log: list[object], a: int, b: int) -> object: return r + @compiled def mul2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a * b @@ -84,6 +90,7 @@ def mul2(log: list[object], a: int, b: int) -> object: return r + @compiled def neg2(log: list[object], a: int) -> object: log.append("start") r: int = -a @@ -93,6 +100,7 @@ def neg2(log: list[object], a: int) -> object: return r + @compiled def floordiv2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -104,6 +112,7 @@ def floordiv2(log: list[object], a: int, b: int) -> object: return r + @compiled def mod2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - 1 @@ -115,6 +124,7 @@ def mod2(log: list[object], a: int, b: int) -> object: return r + @compiled def lsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -122,6 +132,7 @@ def lsh(log: list[object], a: int, b: int) -> object: return r + @compiled def rsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -131,6 +142,7 @@ def rsh(log: list[object], a: int, b: int) -> object: return r + @compiled def roundtrip(log: list[object], a: int) -> object: log.append(a) x: int = a @@ -145,6 +157,7 @@ def roundtrip(log: list[object], a: int) -> object: return x + @compiled def compare(log: list[object], a: int, b: int) -> None: log.append("start") x: int = a + 1 @@ -167,6 +180,7 @@ def compare(log: list[object], a: int, b: int) -> None: log.append("y-zero") + @compiled def fact(log: list[object], n: int) -> object: log.append(n) f: int = 1 @@ -176,6 +190,7 @@ def fact(log: list[object], n: int) -> object: return f + @compiled def grow(log: list[object], a: int, k: int) -> object: x: int = a i: int = 0 @@ -186,16 +201,19 @@ def grow(log: list[object], a: int, k: int) -> object: return x + @compiled def consts(log: list[object]) -> None: log.append(7000000000) log.append(5) log.append(-7000000000) + @compiled def next_state(s: int) -> int: return (s * 48271) % MOD + @compiled def drive(log: list[object], seed: int, n: int) -> object: s: int = seed + 1 for _ in range(n): @@ -204,37 +222,44 @@ def drive(log: list[object], seed: int, n: int) -> object: return s + @compiled def read_limit(log: list[object]) -> object: log.append(1) return LIMIT + @compiled def read_twice(log: list[object], cb: object) -> None: log.append(LIMIT) cb() log.append(LIMIT) + @compiled def use_builtin(log: list[object], s: str) -> object: log.append(1) return len(s) + @compiled def read_fn(log: list[object]) -> object: log.append(1) return GFN + @compiled def read_type(log: list[object]) -> object: log.append(1) return GTYPE + @compiled def read_mod(log: list[object]) -> object: log.append(1) return GMOD + @compiled def call_g(x: object) -> object: return GFN() ''') @@ -314,7 +339,7 @@ def test_int_locals_of_impure_functions_are_native(built): def test_an_int_fed_from_an_object_stays_an_object(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def f(log: list[object], g: object) -> object: log.append(1) n: int = g() @@ -656,7 +681,7 @@ def test_a_cache_that_is_never_invalidated_is_caught(tmp_path): def test_an_int_loop_is_native_work_for_the_cost_rule(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def digits(log: list[object], n: int) -> None: x: int = n + 1 while x > 0: @@ -672,7 +697,7 @@ def digits(log: list[object], n: int) -> None: def test_an_object_only_loop_is_still_rejected(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def words(xs: list[object], ys: list[object]) -> None: for x in xs: ys.append(x) @@ -688,7 +713,7 @@ def words(xs: list[object], ys: list[object]) -> None: @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) path = d / "wordfreq.py" path.write_text(src) proved, diags = verify.verify(frontend.lower(path)) diff --git a/python-multiplatform-ksp/src/test/python/test_osr_exit.py b/python-multiplatform-ksp/src/test/python/test_osr_exit.py index fbfbdd3cf..227d50ecd 100644 --- a/python-multiplatform-ksp/src/test/python/test_osr_exit.py +++ b/python-multiplatform-ksp/src/test/python/test_osr_exit.py @@ -17,11 +17,11 @@ from typedpython import cbuild, cgen, frontend, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled SCALE = 2.0 + @compiled def scale_all(x: list[float]) -> float: s = 0.0 for i in range(len(x)): @@ -30,6 +30,7 @@ def scale_all(x: list[float]) -> float: return s + @compiled def nested(x: list[float], n: int) -> int: hits = 0 for r in range(n): @@ -43,6 +44,7 @@ def nested(x: list[float], n: int) -> int: return hits + @compiled def countdown(x: list[int]) -> int: k = 0 total = 0 diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index 55fc184c6..8f215ecd2 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -1,14 +1,14 @@ -"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277). +"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277, #358). -Step 1 scope: in a function marked by the comment line `# typedpython: parallel` directly above its -`def`, a `for i in range(len(a))` loop whose body is only stores `a[i] = ` (the same +Step 1 scope: in a function decorated with `@compiled(parallel=True)`, a `for i in range(len(a))` +loop whose body is only stores `a[i] = ` (the same index i, each proven in bounds), over list copy-in or local arrays, with values built from constants, loop-invariant scalars, `float(i)`, reads `b[i]` at the same i (proven) and F64 + - * and negation, runs on several C threads above TP_PAR_MIN_WORK. The verifier recomputes the set (`ForRange.par`) and rejects a declared one that differs (`verify/par-loop`). Everything else compiles as before. Red record: written before the implementation. Before it, `ir.ForRange` had no `par`, `ir.Function` -had no `parallel`, the marker comment was ignored and the generated module had no +had no `parallel`, the parallel opt-in was ignored and the generated module had no `__typedpython_parallel__`; every test below that touches those failed (see the report in #277). """ from __future__ import annotations @@ -25,6 +25,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, ir, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -33,39 +34,39 @@ PAR_RULE = "verify/par-loop" SOURCE = '''\ -# typedpython: compiled -# typedpython: parallel +@compiled(parallel=True) def scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b +@compiled def scale_unmarked(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b -# typedpython: parallel +@compiled(parallel=True) def ramp(xs: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = -(float(i) * a) + xs[i] -# typedpython: parallel +@compiled(parallel=True) def two_stores(xs: list[float], ys: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = xs[i] + a xs[i] = xs[i] * xs[i] -# typedpython: parallel +@compiled(parallel=True) def neighbour(xs: list[float]) -> None: for i in range(len(xs) - 1): xs[i] = xs[i + 1] -# typedpython: parallel +@compiled(parallel=True) def total(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -73,23 +74,24 @@ def total(xs: list[float]) -> float: return s +@compiled def helper(x: float) -> float: return x * 2.0 -# typedpython: parallel +@compiled(parallel=True) def with_call(xs: list[float]) -> None: for i in range(len(xs)): xs[i] = helper(xs[i]) -# typedpython: parallel +@compiled(parallel=True) def divides(xs: list[float], d: float) -> None: for i in range(len(xs)): xs[i] = xs[i] / d -# typedpython: parallel +@compiled(parallel=True) def local_fill(a: float) -> float: t = [0.0] * 300000 for i in range(len(t)): @@ -97,7 +99,7 @@ def local_fill(a: float) -> float: return t[7] + t[299999] -# typedpython: parallel +@compiled(parallel=True) def local_dyn(n: int, a: float) -> float: t = [1.0] * n for i in range(len(t)): @@ -239,7 +241,7 @@ def test_c_has_a_region_only_for_eligible_loops(built): def test_no_par_code_without_a_marker(tmp_path): - src = SOURCE.replace("# typedpython: parallel\n", "") + src = SOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="nomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -325,7 +327,7 @@ def test_list_untouched_when_the_call_raises_before_the_loop(mod): # --- child processes: thread counts, creation failure, concurrency ----------------------------------- -CHILD = """\ +CHILD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, struct, sys, threading path = sys.argv[1] name = path.replace(chr(92), "/").rsplit("/", 1)[-1].split(".")[0] @@ -438,9 +440,8 @@ def sh(): # ===================================================================================================== RSOURCE = '''\ -# typedpython: compiled -# typedpython: parallel +@compiled(parallel=True) def r_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -448,7 +449,7 @@ def r_sum(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def r_sum_init(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -456,7 +457,7 @@ def r_sum_init(xs: list[int], s0: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def r_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -464,7 +465,7 @@ def r_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def r_sum_mul(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -472,7 +473,7 @@ def r_sum_mul(xs: list[int], c: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def r_two(xs: list[int]) -> int: a: int = 0 b: int = 7 @@ -482,7 +483,7 @@ def r_two(xs: list[int]) -> int: return a * 3 + b -# typedpython: parallel +@compiled(parallel=True) def r_range_sum(xs: list[int]) -> int: s: int = 5 for i in range(len(xs)): @@ -490,7 +491,7 @@ def r_range_sum(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def r_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -501,6 +502,7 @@ def r_local(n: int) -> int: return s +@compiled def r_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -508,7 +510,7 @@ def r_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def n_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -516,7 +518,7 @@ def n_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def n_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -525,7 +527,7 @@ def n_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def n_float(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -533,7 +535,7 @@ def n_float(xs: list[float]) -> float: return s -# typedpython: parallel +@compiled(parallel=True) def n_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -543,7 +545,7 @@ def n_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel +@compiled(parallel=True) def n_mixed(xs: list[int]) -> int: t = [0.0] * len(xs) s: int = 0 @@ -553,7 +555,7 @@ def n_mixed(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def n_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -561,7 +563,7 @@ def n_div(xs: list[int], d: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def n_shift_step(xs: list[int]) -> int: s: int = 0 for i in range(0, len(xs), 2): @@ -839,7 +841,7 @@ def test_reduction_results_identical_for_every_thread_count(rbuilt): def test_no_reduction_region_without_a_marker(tmp_path): - src = RSOURCE.replace("# typedpython: parallel\n", "") + src = RSOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="rnomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -859,9 +861,8 @@ def test_no_reduction_region_without_a_marker(tmp_path): # ===================================================================================================== MSOURCE = '''\ -# typedpython: compiled -# typedpython: parallel +@compiled(parallel=True) def mn_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -870,7 +871,7 @@ def mn_i(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def mx_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -879,7 +880,7 @@ def mx_i(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def mn_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -888,7 +889,7 @@ def mn_f(xs: list[float]) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def mx_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -897,7 +898,7 @@ def mx_f(xs: list[float]) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def mn_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -906,7 +907,7 @@ def mn_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def mx_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -915,7 +916,7 @@ def mx_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def mn_i_expr(xs: list[int], c: int) -> int: m = c for i in range(len(xs)): @@ -924,7 +925,7 @@ def mn_i_expr(xs: list[int], c: int) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def mn_f_expr(xs: list[float], a: float) -> float: m = a for i in range(len(xs)): @@ -933,7 +934,7 @@ def mn_f_expr(xs: list[float], a: float) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def mn_and_sum(xs: list[int]) -> int: m = xs[0] s = 0 @@ -944,7 +945,7 @@ def mn_and_sum(xs: list[int]) -> int: return m * 1000003 + s -# typedpython: parallel +@compiled(parallel=True) def two_ext(xs: list[float]) -> float: lo = xs[0] hi = xs[0] @@ -956,7 +957,7 @@ def two_ext(xs: list[float]) -> float: return hi - lo -# typedpython: parallel +@compiled(parallel=True) def mn_i_init(xs: list[int], init: int) -> int: m = init for i in range(len(xs)): @@ -965,7 +966,7 @@ def mn_i_init(xs: list[int], init: int) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def mn_i_big_e(xs: list[int], c: int) -> int: m = 0 for i in range(len(xs)): @@ -974,6 +975,7 @@ def mn_i_big_e(xs: list[int], c: int) -> int: return m +@compiled def mn_unmarked(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -982,7 +984,7 @@ def mn_unmarked(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def n_le(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -991,7 +993,7 @@ def n_le(xs: list[float]) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def n_other_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1000,7 +1002,7 @@ def n_other_value(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def n_else(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1012,7 +1014,7 @@ def n_else(xs: list[int]) -> int: return m + s -# typedpython: parallel +@compiled(parallel=True) def n_two_in_then(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1023,7 +1025,7 @@ def n_two_in_then(xs: list[int]) -> int: return m + s -# typedpython: parallel +@compiled(parallel=True) def n_acc_in_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1033,7 +1035,7 @@ def n_acc_in_value(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def n_cond_other(xs: list[int], t: int) -> int: m = xs[0] for i in range(len(xs)): @@ -1042,7 +1044,7 @@ def n_cond_other(xs: list[int], t: int) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def n_mixed_types(xs: list[int], init: float) -> float: m = init for i in range(len(xs)): @@ -1051,7 +1053,7 @@ def n_mixed_types(xs: list[int], init: float) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def n_acc_reads_acc(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1062,7 +1064,7 @@ def n_acc_reads_acc(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def n_div(xs: list[float], d: float) -> float: m = xs[0] for i in range(len(xs)): @@ -1490,9 +1492,8 @@ def spin(): # ===================================================================================================== BSOURCE = '''\ -# typedpython: compiled -# typedpython: parallel +@compiled(parallel=True) def b_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1500,7 +1501,7 @@ def b_and(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1508,7 +1509,7 @@ def b_or(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1516,7 +1517,7 @@ def b_xor(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_and_swapped(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -1524,7 +1525,7 @@ def b_and_swapped(xs: list[int], s0: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_or_swapped(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1532,7 +1533,7 @@ def b_or_swapped(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_xor_expr(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1540,7 +1541,7 @@ def b_xor_expr(xs: list[int], c: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_or_not(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1548,7 +1549,7 @@ def b_or_not(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def b_mix(xs: list[int]) -> int: a: int = 0 x: int = 5 @@ -1563,7 +1564,7 @@ def b_mix(xs: list[int]) -> int: return a + x * 3 + o * 5 + m * 7 -# typedpython: parallel +@compiled(parallel=True) def b_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -1574,6 +1575,7 @@ def b_local(n: int) -> int: return s +@compiled def b_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1581,7 +1583,7 @@ def b_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1589,7 +1591,7 @@ def nb_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1598,7 +1600,7 @@ def nb_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_and_then_add(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1607,7 +1609,7 @@ def nb_and_then_add(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_shift(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1615,7 +1617,7 @@ def nb_shift(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_rshift(xs: list[int], k: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1623,7 +1625,7 @@ def nb_rshift(xs: list[int], k: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def nb_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -1633,7 +1635,7 @@ def nb_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel +@compiled(parallel=True) def nb_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1794,15 +1796,14 @@ def test_bitwise_equals_interpreter_for_every_thread_count(bbuilt, threads): # ===================================================================================================== TSOURCE = '''\ -# typedpython: compiled -# typedpython: parallel +@compiled(parallel=True) def t_scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b -# typedpython: parallel +@compiled(parallel=True) def t_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1810,7 +1811,7 @@ def t_sum(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def t_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -1818,7 +1819,7 @@ def t_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def t_min(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1827,7 +1828,7 @@ def t_min(xs: list[int]) -> int: return m -# typedpython: parallel +@compiled(parallel=True) def t_max(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1836,7 +1837,7 @@ def t_max(xs: list[float]) -> float: return m -# typedpython: parallel +@compiled(parallel=True) def t_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1844,7 +1845,7 @@ def t_and(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def t_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1852,7 +1853,7 @@ def t_or(xs: list[int]) -> int: return s -# typedpython: parallel +@compiled(parallel=True) def t_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 6efbeb14d..a50a4bfe5 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -20,9 +20,9 @@ def interpreted(path: Path): SCALAR = """\ -# typedpython: compiled +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -30,6 +30,7 @@ def total(n: int) -> float: return acc +@compiled def fact(n: int) -> int: r: int = 1 for k in range(2, n + 1): @@ -53,6 +54,7 @@ def test_scalar_module_compiles_and_matches_cpython(write, tmp_path): def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): path = write("mixed.py", SCALAR + """ +@compiled def greet(name: str) -> str: return name[1:] """) @@ -68,16 +70,16 @@ def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path) def test_gate_errors_block_compilation(write, tmp_path): - path = write("bad.py", "# typedpython: compiled\n\ndef f(x):\n return x\n") + path = write("bad.py", "@compiled\ndef f(x):\n return x\n") with pytest.raises(pipeline.CompileError, match="implicit-any-parameter"): pipeline.compile_module(path, tmp_path / "out") DEMO = """\ -# typedpython: compiled import sys +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -85,6 +87,7 @@ def total(n: int) -> float: return acc +@compiled def main() -> None: print(f"{total(int(sys.argv[1])):.3f}") """ @@ -103,15 +106,16 @@ def test_demo_reports_identical_output(write, tmp_path): ARRAY_CALLBACK = """\ -# typedpython: compiled from typing import Callable +@compiled def first_after(xs: list[float], cb: Callable[[], None]) -> float: cb() return xs[0] +@compiled def fill_then(xs: list[float], cb: Callable[[], None]) -> None: xs[1] = 5.0 cb() @@ -138,9 +142,9 @@ def cb2(): GC_REACHABLE = """\ -# typedpython: compiled +@compiled class Cell: a: object b: object @@ -151,10 +155,12 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def fresh(a: object, n: int) -> object: c = Cell(None, None) seta(c, a) @@ -179,3 +185,153 @@ def test_a_fresh_local_does_not_exempt_the_callee_class_guard(tmp_path): c = mod.Cell(None, None) mod.seta(c, 7) assert c.a == 7 + + +def test_default_fp_flags_stay_exact_and_fastmath_is_per_function_only(write, tmp_path): + """#153 / #332: by default, `-ffp-contract=off` and `-fno-fast-math` on GCC/Clang and + `/fp:precise` on MSVC are pinned so floating-point arithmetic is bit-identical to CPython. + `@compiled(fastmath=True)` opts in for that function only (not CPython-identical), leaving + sibling `@compiled` functions in the same module exact.""" + from typedpython import cbuild, cgen, frontend, verify + + assert "-ffp-contract=off" in cbuild.DEFAULT_FLAGS + assert "-fno-fast-math" in cbuild.DEFAULT_FLAGS + assert "/fp:precise" in cbuild.MSVC_DEFAULT_FLAGS + assert not any("fast" in f for f in cbuild.MSVC_DEFAULT_FLAGS) + + path = write( + "fp_mod.py", + """ + @compiled + def fma_exact(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled(fastmath=True) + def fma_fast(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled + def calls_fast_and_adds(a: float, b: float, c: float) -> float: + # caller is exact: its own `a * b + c` must not contract even though it calls `fma_fast` + _ = fma_fast(1.0, 2.0, 3.0) + return a * b + c + """, + ) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + assert "#pragma STDC FP_CONTRACT OFF" in c_text + assert "\nTP_FASTMATH_BEGIN\n" in c_text and "\nTP_FASTMATH_END\n" in c_text + # Only `fma_fast` is wrapped in standalone TP_FASTMATH_BEGIN / TP_FASTMATH_END lines: + assert c_text.count("\nTP_FASTMATH_BEGIN\n") == 1 + assert c_text.count("\nTP_FASTMATH_END\n") == 1 + + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + mod = pipeline.load(res) + ref = interpreted(path) + # 1 + 2**-27 squared: exact double product rounds to 1 + 2**-26, while FMA(a, a, -1.0) keeps 2**-54 + a = 1.0 + 2.0**-27 + c = -1.0 + want_exact = ref.fma_exact(a, a, c) + assert mod.fma_exact(a, a, c) == want_exact + assert mod.calls_fast_and_adds(a, a, c) == want_exact + # fma_fast runs and returns a float (on targets with FMA it equals 2**-26 + 2**-54; either is valid under fastmath) + got_fast = mod.fma_fast(a, a, c) + assert isinstance(got_fast, float) + + + +_NAN_SRC = """ +@compiled(fastmath=True) +def nan_count(xs: list[float]) -> int: + n = 0 + for i in range(len(xs)): + x = xs[i] + if x != x: + n += 1 + return n + +@compiled(fastmath=True) +def is_nan(x: float) -> bool: + return x != x +""" + + +def _real_gcc(): + """A GCC driver that is really GCC (macOS `gcc` is clang), or None.""" + import shutil + import subprocess + for name in ("gcc", "gcc-15", "gcc-14", "gcc-13"): + exe = shutil.which(name) + if exe: + out = subprocess.run([exe, "--version"], capture_output=True, text=True).stdout + if "clang" not in out.lower(): + return exe + return None + + +def test_gcc_fastmath_scope_is_reassociation_and_contract_only(write): + """#368: GCC's fastmath macro allows the same scope as clang and MSVC, reassociation and FMA + contraction (plus the no-signed-zeros / no-trapping-math that associative-math needs); never + `fast-math`, whose finite-math-only folds `x != x` to false.""" + from typedpython import cgen, frontend, verify + + path = write("fm_scope.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + gcc_line = next(l for l in c_text.splitlines() if "define TP_FASTMATH_BEGIN" in l and "GCC" in l) + assert "fast-math" not in gcc_line + for opt in ("associative-math", "no-signed-zeros", "no-trapping-math", "fp-contract=fast"): + assert opt in gcc_line, (opt, gcc_line) + assert "finite-math" not in gcc_line + + +def test_fastmath_keeps_nan_checks(write, tmp_path): + """#368: `x != x` inside a fastmath function still detects NaN (host compiler).""" + path = write("fm_nan.py", _NAN_SRC) + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True and mod.is_nan(1.0) is False + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_fastmath_keeps_nan_checks_under_gcc(write, tmp_path, monkeypatch): + """#368: the same under a real GCC (Linux CI); skipped where no GCC is installed.""" + import sys + from typedpython import cbuild + gcc = _real_gcc() + if gcc is None or sys.platform == "win32": + pytest.skip("no real GCC on this host") + monkeypatch.setattr(cbuild, "compiler", lambda: [gcc]) + path = write("fm_nan_gcc.py", _NAN_SRC) + # GCC only warns on an optimize option it does not know; make that an error so a dropped + # option cannot pass silently. + from typedpython import cgen, frontend, verify + proved, _ = verify.verify(frontend.lower(path)) + c_file = tmp_path / "fm_nan_gcc.c" + c_file.write_text(cgen.generate(proved, path)) + chk = subprocess.run([gcc, "-fsyntax-only", "-Werror=pragmas", "-Werror=attributes", + f"-I{cbuild._python_include(None)}", f"-I{cbuild.RUNTIME_DIR}", str(c_file)], + capture_output=True, text=True) + assert chk.returncode == 0, chk.stderr + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_msvc_fastmath_scope_is_contract_only(write): + """#368: cl.exe has no reassociation-only switch; `float_control(precise, off)` is /fp:fast and + folds `x != x` (seen on Windows CI). MSVC's fastmath macro enables FMA contraction only.""" + from typedpython import cgen, frontend, verify + + path = write("fm_msvc.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + lines = cgen.generate(proved, path).splitlines() + i = lines.index("#elif defined(_MSC_VER)", next(k for k, l in enumerate(lines) if "TP_FASTMATH_BEGIN" in l)) + begin = next(l for l in lines[i:] if "define TP_FASTMATH_BEGIN" in l) + assert "fp_contract(on)" in begin, begin + assert "float_control" not in begin and "precise" not in begin, begin diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py index dd7cfad86..3735df9bc 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -9,25 +9,24 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, frontend, pipeline -HEADER = "# typedpython: compiled\n" - -B = HEADER + textwrap.dedent("""\ +B = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 - """) -A = HEADER + textwrap.dedent("""\ + """)) +A = decorate_compiled(textwrap.dedent("""\ import b def inc(n: int) -> int: return n + 1 - """) -C = HEADER + textwrap.dedent("""\ + """)) +C = decorate_compiled(textwrap.dedent("""\ def half(x: float) -> float: return x * 0.5 - """) + """)) def interpreted(path: Path): @@ -133,7 +132,7 @@ def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): root = tmp_path / "proj" root.mkdir() - (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s[1:]\n") + (root / "plain.py").write_text("@compiled\ndef f(s: str) -> str:\n return s[1:]\n") cache = tmp_path / "cache" first = pipeline.compile_project([root / "plain.py"], root, cache) m = first.modules[0] @@ -151,7 +150,7 @@ def test_interface_describes_the_verified_module_not_the_lowered_one(tmp_path, m root = tmp_path / "proj" root.mkdir() - (root / "dep.py").write_text(HEADER + "def ok(n: int) -> int:\n return n + 1\n") + (root / "dep.py").write_text("@compiled\ndef ok(n: int) -> int:\n return n + 1\n") real_lower = frontend.lower seen = {} diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index 289e1de66..84c36f1e6 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -15,20 +15,22 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" SOURCE = """\ -# typedpython: compiled +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def ack(m: int, n: int) -> int: if m == 0: return n + 1 @@ -37,30 +39,35 @@ def ack(m: int, n: int) -> int: return ack(m - 1, ack(m, n - 1)) +@compiled def is_even(n: int) -> bool: if n == 0: return True return is_odd(n - 1) +@compiled def is_odd(n: int) -> bool: if n == 0: return False return is_even(n - 1) +@compiled def down(n: int) -> int: if n == 0: return 0 return 1 + down(n - 1) +@compiled def pow2(n: int) -> int: if n == 0: return 1 return 2 * pow2(n - 1) +@compiled def fact(n: int) -> int: if n < 2: return 1 @@ -233,7 +240,7 @@ def inner(j): # --- sanitizers: the RecursionError path --------------------------------------------------------- -ASAN_SCRIPT = textwrap.dedent('''\ +ASAN_SCRIPT = COMPILED_SHIM + textwrap.dedent('''\ import sys sys.path.insert(0, {pkg!r}) from typedpython import cbuild diff --git a/python-multiplatform-ksp/src/test/python/test_regions.py b/python-multiplatform-ksp/src/test/python/test_regions.py index 0246fd207..7d7080ee9 100644 --- a/python-multiplatform-ksp/src/test/python/test_regions.py +++ b/python-multiplatform-ksp/src/test/python/test_regions.py @@ -25,6 +25,7 @@ import pytest from test_cgen import runtime_dir, need_runtime +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, cost, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -37,12 +38,12 @@ FAIL_AFTER = "__typedpython_region_fail_after__" # (n) -> the nth region object allocation fails; -1 off SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython region test module.""" KEPT = None + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -53,12 +54,14 @@ def __init__(self, left: "Node | None", right: "Node | None") -> None: self.right = right + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check(node: Node) -> int: left: Node | None = node.left right: Node | None = node.right @@ -67,10 +70,12 @@ def check(node: Node) -> int: return 1 + check(left) + check(right) + @compiled def tree_sum(d: int) -> int: return check(make(d)) + @compiled def repeat(d: int, n: int) -> int: t: int = 0 for _ in range(n): @@ -78,35 +83,42 @@ def repeat(d: int, n: int) -> int: return t + @compiled def escapes(d: int) -> Node: return make(d) + @compiled def keeps(d: int) -> int: global KEPT KEPT = make(d) return check(KEPT) + @compiled def prints(d: int) -> int: n = make(d) print("unknown code while the tree is alive") return check(n) + @compiled def ident(d: int) -> int: return id(make(d)) * 0 + @compiled def same(d: int) -> bool: return make(d) is make(d) + @compiled def mixed(d: int) -> int: t = make(d) return check(make(d)) + check(t) + @compiled def noisy(d: int) -> object: print("unknown code") return check(make(d)) @@ -381,10 +393,11 @@ def test_a_forged_module_level_variant_set_is_reported(built): def test_a_finalizer_or_a_field_store_keeps_the_class_out_of_a_region(tmp_path): - base = SOURCE.split("def escapes")[0] + base = SOURCE.split("@compiled\ndef escapes")[0] with_del = base.replace(" self.right = right\n", " self.right = right\n\n def __del__(self) -> None:\n pass\n", 1) with_store = base + textwrap.dedent('''\ + @compiled def poke(node: Node) -> int: node.left = None return 0 diff --git a/python-multiplatform-ksp/src/test/python/test_resume.py b/python-multiplatform-ksp/src/test/python/test_resume.py index 14115cc79..984c7b4b3 100644 --- a/python-multiplatform-ksp/src/test/python/test_resume.py +++ b/python-multiplatform-ksp/src/test/python/test_resume.py @@ -519,7 +519,7 @@ def lower(tmp_path, source): def test_frontend_gives_a_snapshot_holder_one_resume_per_loop(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def f(xs: list[float], n: int) -> float: t = 0.0 for i in range(n): @@ -572,7 +572,7 @@ def f(xs: list[float], n: int) -> float: def test_frontend_function_without_a_snapshot_has_no_resumes(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def g(n: int) -> int: t = 0 for i in range(n): diff --git a/python-multiplatform-ksp/src/test/python/test_str_dict.py b/python-multiplatform-ksp/src/test/python/test_str_dict.py index 2362033c4..e6bc52c7b 100644 --- a/python-multiplatform-ksp/src/test/python/test_str_dict.py +++ b/python-multiplatform-ksp/src/test/python/test_str_dict.py @@ -26,70 +26,84 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled from typing import SupportsIndex + @compiled def get(d: dict[object, object], k: object) -> object: return d[k] + @compiled def get_list(xs: list[object], i: int) -> object: return xs[i] + @compiled def get_index(xs: list[object], i: SupportsIndex) -> object: return xs[i] + @compiled def get_str(s: str, i: int) -> str: return s[i] + @compiled def get_nested(d: dict[object, dict[object, object]], a: object, b: object) -> object: return d[a][b] + @compiled def put(d: dict[object, object], k: object, v: object) -> None: d[k] = v + @compiled def put_list(xs: list[object], i: int, v: object) -> None: xs[i] = v + @compiled def has(c: list[object], x: object) -> bool: return x in c + @compiled def has_not(c: list[object], x: object) -> bool: return x not in c + @compiled def has_dict(c: dict[object, object], x: object) -> bool: return x in c + @compiled def has_str(c: str, x: str) -> bool: return x in c + @compiled def has_branch(c: list[object], x: object) -> str: if x in c: return "yes" return "no" + @compiled def has_chain(c: list[object], x: object, y: object) -> bool: return x in c in y + @compiled def pairs(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b in xs: @@ -98,6 +112,7 @@ def pairs(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def triples(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b, c in xs: @@ -106,6 +121,7 @@ def triples(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def each(xs: list[object]) -> list[object]: out: list[object] = [] for x in xs: @@ -115,6 +131,7 @@ def each(xs: list[object]) -> list[object]: return out + @compiled def first_after(xs: list[object], stop: object) -> object: found: object = None for x in xs: @@ -124,12 +141,14 @@ def first_after(xs: list[object], stop: object) -> object: return found + @compiled def return_inside(xs: list[object]) -> object: for x in xs: return x return None + @compiled def nested(xs: list[list[object]]) -> list[object]: out: list[object] = [] for row in xs: @@ -140,6 +159,7 @@ def nested(xs: list[list[object]]) -> list[object]: return out + @compiled def items_of(d: dict[str, object]) -> list[object]: out: list[object] = [] for k, v in d.items(): @@ -148,6 +168,7 @@ def items_of(d: dict[str, object]) -> list[object]: return out + @compiled def mutate_while_iterating(d: dict[str, object]) -> object: n: int = 0 for k in d: @@ -156,30 +177,37 @@ def mutate_while_iterating(d: dict[str, object]) -> object: return n + @compiled def fmt(x: object, y: float, n: int) -> str: return f"{x}|{x!r}|{x!s}|{x!a}|{y:.3f}|{n:05d}|{x:>5}|{n}" + @compiled def fmt_one(x: object) -> str: return f"{x}" + @compiled def fmt_text(x: object) -> str: return f"a{x}b{x!r}c" + @compiled def fmt_empty() -> str: return f"" + @compiled def fmt_debug(x: object) -> str: return f"{x=}" + @compiled def cat(a: str, b: str) -> str: return a + b + @compiled def cat_in_place(a: str, b: str) -> str: s: str = "" s += a @@ -187,6 +215,7 @@ def cat_in_place(a: str, b: str) -> str: return s + @compiled def grow(xs: list[object], ys: list[object]) -> list[object]: out: list[object] = [] out += xs @@ -194,6 +223,7 @@ def grow(xs: list[object], ys: list[object]) -> list[object]: return out + @compiled def empties() -> object: d: dict[object, object] = {} xs: list[object] = [] @@ -201,6 +231,7 @@ def empties() -> object: return xs + @compiled class Acc: def __init__(self, log: list[object]) -> None: self.log = log @@ -230,6 +261,7 @@ def __imod__(self, other: int) -> Acc: return self + @compiled def aug_ops(a: Acc, b: int) -> Acc: a += b a -= b @@ -240,12 +272,14 @@ def aug_ops(a: Acc, b: int) -> Acc: return a + @compiled def aug_list(xs: list[str], n: int) -> list[str]: xs *= n xs += xs return xs + @compiled def lookup_default(d: dict[str, int], k: str) -> object: counts: dict[str, int] = {} counts[k] = counts.get(k, 0) + 1 @@ -905,7 +939,7 @@ def run_main(main, n): @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) proved, diags, m = build_source(d, src, name="wordfreq") spec = importlib.util.spec_from_file_location("wordfreq_plain", WORDFREQ) plain = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/test/python/test_tracing.py b/python-multiplatform-ksp/src/test/python/test_tracing.py index 2c6939249..823c1dd1d 100644 --- a/python-multiplatform-ksp/src/test/python/test_tracing.py +++ b/python-multiplatform-ksp/src/test/python/test_tracing.py @@ -16,17 +16,19 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline SOURCE = """\ -# typedpython: compiled from typing import Callable +@compiled def sq(x: float) -> float: return x * x +@compiled def loop(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -34,17 +36,19 @@ def loop(n: int) -> float: return acc +@compiled def leaf(cb: Callable[[], None], n: float) -> float: cb() return n + 1.0 +@compiled def outer(cb: Callable[[], None], n: float) -> float: cb() return leaf(cb, n) + leaf(cb, n) """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys, threading path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("tmod", path) diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index 46e46c00b..a51654833 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -24,7 +24,6 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.konan.target.KonanTarget.* import org.jetbrains.kotlin.konan.target.linker -import python.multiplatform.gradle.CPythonFlavour import python.multiplatform.gradle.CPythonIncludeDirectories import python.multiplatform.gradle.CPythonIncludeLayout import python.multiplatform.gradle.EmbeddedPythonVersion @@ -63,9 +62,21 @@ plugins { val configuredPythonVersion = project.property("pythonVersion").toString() -val pythonFreeThreaded = project.findProperty("pythonFreeThreaded")?.toString()?.toBoolean() ?: false +// Issue #158, maintainer 2026-10-04: "Free-threaded is the default and the only build. The GIL +// build is not kept as an option. Baseline is CPython 3.15t." Desktop (JVM, macosArm64, linuxX64) +// therefore always acquires python-build-standalone's free-threaded build, and the +// `-PpythonFreeThreaded` switch that used to choose it is gone. Android (arm64-v8a, x86_64), +// androidNative and iOS embed the project's own 3.15t builds from the GitHub release +// `cpython-t-mobile` (#328; leader 2026-11-13: no opt-in flag on mobile, 3.15t is the +// only build). Only wasm still embeds a GIL build, until #175. Which build each platform embeds is +// the `embedsFreeThreadedPython` expect/actual in the library, not a build-wide flag. +if (project.findProperty("pythonFreeThreaded") != null) { + throw GradleException( + "-PpythonFreeThreaded was removed (issue #158): desktop always embeds the free-threaded " + + "CPython, and the GIL desktop build is not kept as an option. Remove the property.", + ) +} val pbsRelease = project.findProperty("pythonBuildStandaloneRelease")?.toString() ?: "20261013" -val pythonAppleSupportBuild = project.findProperty("pythonAppleSupportBuild")?.toString() ?: "b11" val pythonVersion = configuredPythonVersion val libraryVersion = "$pythonVersion-${project.property("libraryVersionSuffix")}" // the root's publishedVersion (#70) @@ -78,32 +89,72 @@ buildkonfig { defaultConfigs { buildConfigField(FieldSpec.Type.STRING, "pythonVersion", pythonVersion) buildConfigField(FieldSpec.Type.STRING, "libraryVersion", libraryVersion) - // The free-threaded build is not merely a differently-compiled binary: it renames every - // artefact the runtime looks up by name. The shared library is `libpython3.14t.dylib`, not - // `libpython3.14.dylib`, and the stdlib sits in `lib/python3.14t/`. Without this flag - // reaching Kotlin, `manager.loadLibPython` asks for a file that a free-threaded install - // does not contain, and the failure looks like a missing download rather than a flavour - // mismatch. See `Versions.abiFlags`. - buildConfigField(FieldSpec.Type.BOOLEAN, "pythonFreeThreaded", pythonFreeThreaded.toString()) + // No free-threaded field: which build a platform embeds differs by platform (#158), so it + // is the `embedsFreeThreadedPython` expect/actual (commonMain `EmbeddedPythonBuild.kt`), + // which `Versions.abiFlags` reads. One BuildConfig field shared by every target could not + // say "every native platform 3.15t, wasm GIL". } } val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".") +/** ABI flags of the desktop CPython (macOS, Linux, Windows; JVM and Kotlin/Native): `t`, free-threaded (#158). */ +val desktopAbiSuffix = "t" + +/** + * ABI flags of the Android, androidNative and iOS CPython: `t`. They embed the project's own + * free-threaded builds (#163, #328); there is no GIL mobile build (#158). + */ +val mobileAbiSuffix = "t" + +/** `3.15t`: the name of the mobile `libpython3.15t.so`, `lib/python3.15t/` and `include/python3.15t/`. */ +val mobileTag = "$libVersion$mobileAbiSuffix" + +/** + * Where the project's own free-threaded Android and iOS archives are published (#163, + * `build-ft-mobile.yml`): the GitHub release `cpython-t-mobile`, because neither + * python.org nor BeeWare publishes a free-threaded mobile build. The plugin's iOS consumer wiring + * (`FreeThreadedMobileRelease` in `IosConsumerWiring.kt`) reads the same release. + */ +val freeThreadedMobileReleaseUrl = + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-${configuredPythonVersion}t-mobile" + +/** + * Suffix of the mobile lock keys (`android-aarch64-3.15.0-freethreaded`, `ios-3.15.0-freethreaded`) + * and of the Android extraction directories. The suffix is in the directory name for the same + * reason as [desktopFlavourSuffix]: an `android-aarch64/` GIL tree unpacked by an older checkout + * must never be mistaken for the free-threaded one. + */ +val mobileFlavourSuffix = "-freethreaded" + +/** Lock-key suffix of the mobile archives in `python-checksums.properties`. */ +val mobileLockKeySuffix = "-freethreaded" /** - * The `ftp/python//` directory holding a release, which drops any pre-release suffix: - * `3.15.0rc1` is published under `3.15.0`. Identical to the version itself for a final release, - * so it is always the right thing to build a python.org URL from. + * Fails the build when an extracted mobile CPython is not a free-threaded build: every `pyconfig` + * header in it must define `Py_GIL_DISABLED 1`. The archive name and the lockfile say what was + * meant to be downloaded; this says what was unpacked. * - * Declared here, above every consumer, because both the Android and the iOS download tasks need - * it. It used to sit below the Android tasks, which therefore built their URLs from the raw - * version and 404'd on any pre-release -- `ftp/python/3.15.0rc1/` does not exist. That stayed - * invisible because the archives were already sitting in the download directory, so the fetch was - * skipped; adding the `.sigstore` fetch is what surfaced it. + * The iOS framework's `Headers/pyconfig.h` is a 107-byte wrapper that includes + * `pyconfig-arm64.h` / `pyconfig-x86_64.h` by architecture (observed in the 3.15t archive), so a + * header that only includes another `pyconfig-*` is skipped and the per-architecture ones checked. */ -val pythonOrgReleaseDir = pythonVersion.split(".").let { parts -> - val patch = parts.getOrNull(2)?.takeWhile { it.isDigit() } ?: "0" - "${parts[0]}.${parts[1]}.$patch" +fun requireFreeThreadedHeaders(label: String, extracted: File) { + val name = Regex("""pyconfig(-[A-Za-z0-9_]+)?\.h""") + val wrapper = Regex("""#\s*include\s+["<]pyconfig-""") + val configs = extracted.walkTopDown() + .filter { it.isFile && name.matches(it.name) } + .filterNot { wrapper.containsMatchIn(it.readText()) } + .toList() + if (configs.isEmpty()) throw GradleException("$label: no pyconfig header under $extracted, cannot verify that it is free-threaded") + val re = Regex("""^\s*#\s*define\s+Py_GIL_DISABLED\s+1\b""", RegexOption.MULTILINE) + for (h in configs) { + if (!re.containsMatchIn(h.readText())) { + throw GradleException( + "$label: $h does not define Py_GIL_DISABLED 1. Android and iOS embed only the free-threaded " + + "CPython ${mobileTag} (#158, #328).", + ) + } + } } println("----------------------------------------------------------------------------------------") @@ -113,7 +164,6 @@ println() val licensePath = "src/nativeInterop/cinterop/license" val libPath = "src/nativeInterop/cinterop/lib" -val libPathForDesktop = "$libPath/desktop" val downloadDir = layout.buildDirectory.dir("python-standalone").get().asFile @@ -131,13 +181,14 @@ val downloadDir = layout.buildDirectory.dir("python-standalone").get().asFile val extractedDir = file("$downloadDir/extracted/$configuredPythonVersion") /** - * Suffix that keeps the two desktop flavours apart within [extractedDir]. + * Suffix of the desktop trees within [extractedDir]. * - * The GIL and free-threaded builds of the *same* version are different trees containing - * differently-named libraries (`libpython3.14.dylib` vs `libpython3.14t.dylib`), so they need - * separating for the same reason the version does. + * Desktop acquires only the free-threaded build (#158). The suffix stays in the path because the + * GIL and free-threaded builds of the *same* version are different trees containing + * differently-named libraries (`libpython3.15.dylib` vs `libpython3.15t.dylib`), and a GIL tree + * unpacked by an older checkout must never be mistaken for the free-threaded one. */ -val desktopFlavourSuffix = if (pythonFreeThreaded) "-freethreaded" else "" +val desktopFlavourSuffix = "-freethreaded" /** * Fails the build when the extracted CPython's own headers report a version other than @@ -196,13 +247,13 @@ fun verifyChecksum(key: String, archive: File) { } // ================================================================================================= -// Sigstore verification of the python.org archives (ROADMAP §12, "Download integrity"). +// Sigstore verification of the downloaded CPython archives (ROADMAP §12, "Download integrity"). // // This is a SECOND gate, not a replacement for `verifyChecksum` above. The two prove different // things and neither implies the other: // // the lockfile "these are the exact bytes this repository reviewed and pinned" -// Sigstore "these are the bytes the CPython release manager actually signed" +// Sigstore "these are the bytes the publisher's release workflow actually signed" // // The lockfile is also the only one of the two that works offline, and the only one that covers // every source. So it stays unconditional and Sigstore is layered on top of it. @@ -219,21 +270,18 @@ fun verifyChecksum(key: String, archive: File) { // // ### What is covered, and what is not // -// python.org Android aarch64/x86_64 COVERED -- sibling `.sigstore` bundle -// python.org iOS XCframework (3.15+) COVERED -- same -// astral-sh/python-build-standalone NOT covered. It publishes no sibling signature at all: -// 853 release assets, and the only non-archive among them -// is `SHA256SUMS` (checked directly against the release). -// Its provenance lives in GitHub's attestations API, keyed -// by artifact *digest* rather than filename, and that API -// is rate-limited to 60 requests/hour unauthenticated. That -// is a different mechanism, not this one. The desktop path -// already verifies against the release's own SHA256SUMS in -// addition to the lockfile. -// beeware/Python-Apple-support NOT coverable. The release publishes five tar.gz assets -// and nothing else -- no checksums, no signatures -- and it -// has no attestations either. The SHA-256 pin is the only -// honest instrument available for iOS <= 3.14. +// astral-sh/python-build-standalone COVERED through GitHub's attestations API, keyed by the +// (desktop) artifact digest (`maybeVerifyPbsSigstore` below). The +// release also publishes `SHA256SUMS`, which the desktop +// path checks unconditionally in addition to the lockfile. +// thisisthepy cpython-t-mobile NOT covered. The project's own free-threaded Android and +// (Android, iOS; #163, #328) iOS archives carry no `.sigstore` bundle; the release has a +// `SHA256SUMS`, and the lockfile pins the same values (checked +// when the pins were written). The SHA-256 pin is the gate. +// +// python.org's Android and iOS archives used to be covered here through their sibling +// `.sigstore` bundles. They were GIL builds, and the library no longer acquires them +// (#158: "The GIL build is not kept as an option"; #328), so that path is gone. val verifySigstore = project.findProperty("verifyPythonSignatures")?.toString()?.toBoolean() ?: false val sigstoreVerifierClasspath: Configuration by configurations.creating { @@ -246,33 +294,9 @@ dependencies { add("sigstoreVerifierClasspath", "dev.sigstore:sigstore-java:2.2.0") } -/** The Fulcio certificate identity a python.org bundle must carry. */ +/** The Fulcio certificate identity a Sigstore bundle must carry. */ data class SigstoreIdentity(val subjectAlternativeName: String, val issuer: String) -/** - * Which release manager signed a given CPython version, per https://www.python.org/download/sigstore/ - * - * This is a map rather than a constant on purpose. The signing identity is a property of the - * *release manager*, and it changes between release series: 3.13 was Thomas Wouters signing via - * Google, 3.14 and 3.15 are Hugo van Kemenade signing via GitHub. A hardcoded identity would keep - * verifying happily on the version it was written for and then either fail confusingly or -- far - * worse -- be loosened by the next person to "just make it pass" on an older one. - * - * An unknown series is a hard failure rather than a skip. A verification step that silently - * declines to verify is the thing this whole section exists to avoid. - */ -fun pythonOrgSigstoreIdentity(version: String): SigstoreIdentity { - val series = version.split(".").take(2).joinToString(".") - return when (series) { - "3.14", "3.15" -> SigstoreIdentity("hugo@python.org", "https://github.com/login/oauth") - "3.12", "3.13" -> SigstoreIdentity("thomas@python.org", "https://accounts.google.com") - else -> throw GradleException( - "No Sigstore signing identity is recorded for CPython $series. Add it from " + - "https://www.python.org/download/sigstore/ rather than disabling verification." - ) - } -} - /** * Verifies `archive` against its sibling `.sigstore` bundle, pinning the signer identity. * @@ -361,17 +385,6 @@ fun verifySigstoreBundle(archive: File, bundleUrl: String?, bundleFile: File, id } } -/** Verifies a python.org archive if `-PverifyPythonSignatures=true`, otherwise says it skipped. */ -fun maybeVerifySigstore(archive: File, archiveUrl: String) { - if (!verifySigstore) return - verifySigstoreBundle( - archive = archive, - bundleUrl = "$archiveUrl.sigstore", - bundleFile = file("$downloadDir/${archive.name}.sigstore"), - identity = pythonOrgSigstoreIdentity(configuredPythonVersion) - ) -} - /** Verifies a python-build-standalone archive using GitHub Attestations API if `-PverifyPythonSignatures=true`. */ fun maybeVerifyPbsSigstore(archive: File, actualHash: String) { if (!verifySigstore) return @@ -463,7 +476,7 @@ val desktopTargets = mapOf( ) val downloadTasks = desktopTargets.map { (platform, pbsTarget) -> - val flavour = if (pythonFreeThreaded) "freethreaded-install_only_stripped" else "install_only_stripped" + val flavour = "freethreaded-install_only_stripped" val assetName = "cpython-$configuredPythonVersion+$pbsRelease-$pbsTarget-$flavour.tar.gz" val url = "https://github.com/astral-sh/python-build-standalone/releases/download/$pbsRelease/$assetName" val archive = file("$downloadDir/$assetName") @@ -538,18 +551,27 @@ val androidTargets = mapOf( "android-x86_64" to "x86_64" ) +/** + * Android embeds the project's own free-threaded CPython 3.15t for both ABIs it supports, + * arm64-v8a and x86_64 (#328; the owner approved the x86_64 3.15t build on 2026-11-13, #396, so + * both ABIs stay supported). The archives are the release `cpython-t-mobile`'s + * `python-t--linux-android.tar.gz`: a `prefix/` with `libpython3.15t.so`, + * `include/python3.15t` and `lib/python3.15t`, the same layout python.org's GIL archives had with + * the `t` added. There is no GIL Android build (#158). + */ val androidDownloadTasks = androidTargets.map { (platform, arch) -> - val url = "https://www.python.org/ftp/python/$pythonOrgReleaseDir/python-$configuredPythonVersion-$arch-linux-android.tar.gz" - val archive = file("$downloadDir/python-$configuredPythonVersion-$arch-linux-android.tar.gz") - val extractDir = file("$extractedDir/$platform") - val lockKey = "$platform-$configuredPythonVersion" + val archiveName = "python-${configuredPythonVersion}t-$arch-linux-android.tar.gz" + val url = "$freeThreadedMobileReleaseUrl/$archiveName" + val archive = file("$downloadDir/$archiveName") + val extractDir = file("$extractedDir/$platform$mobileFlavourSuffix") + val lockKey = "$platform-$configuredPythonVersion$mobileLockKeySuffix" pythonArchiveKeys[lockKey] = archive - + val taskName = "downloadPython_${platform.replace("-", "_")}" tasks.register(taskName) { inputs.property("url", url) outputs.dir(extractDir) - + doLast { if (!archive.exists()) { println("Downloading $url") @@ -560,16 +582,11 @@ val androidDownloadTasks = androidTargets.map { (platform, arch) -> } } } - - // python.org publishes no SHA256SUMS, so the lockfile is what pins these bytes. It also - // publishes a sibling `.sigstore` bundle, which `maybeVerifySigstore` checks - // against the release manager's pinned identity when `-PverifyPythonSignatures=true`. - // (An earlier comment here claimed Sigstore verification was "unreasonable in pure - // Gradle". It is not -- `dev.sigstore:sigstore-java` does it in-process; see the - // section above `updatePythonChecksums` for why it is opt-in rather than always on.) + + // The lockfile is the gate (see "What is covered" above): the release carries no + // Sigstore bundle, and its SHA256SUMS agrees with the pinned values. verifyChecksum(lockKey, archive) - maybeVerifySigstore(archive, url) - + val isEmpty = extractDir.list()?.isEmpty() ?: true if (isEmpty) { println("Extracting $archive to $extractDir") @@ -579,51 +596,34 @@ val androidDownloadTasks = androidTargets.map { (platform, arch) -> } } requireHeaderVersion(platform, extractDir) + requireFreeThreadedHeaders(platform, extractDir) } } } /** - * Which project publishes the iOS `Python.xcframework` for the configured version. + * iOS embeds the project's own free-threaded `Python.xcframework` (#163, #328): the release + * `cpython-t-mobile`'s `python-t-iOS-XCframework.tar.gz`, the archive the + * plugin's iOS consumer wiring also downloads (SPEC T-2a), so a consumer never pairs this klib with + * a different framework. Slices `ios-arm64` and `ios-arm64_x86_64-simulator`, each with + * `Python.framework/Headers` (a `pyconfig.h` that includes `pyconfig-.h`, which defines + * `Py_GIL_DISABLED 1`) and `lib-/python3.15t/`; the pure-Python stdlib is + * `Python.xcframework/lib/python3.15t/`. * - * python.org began publishing an official iOS XCframework with 3.15 (the first entries in - * `ftp/python/3.15.0/` are the 3.15.0b1 betas). BeeWare's Python-Apple-support, which was the - * only source before that, stops at `3.14-b11` (3.14.7) and has no 3.15 tag. The two do not overlap: - * 3.14 and earlier can only come from BeeWare, 3.15 and later only from python.org. So this is a - * hard switch on the version, not a preference. - * - * Swapping is otherwise cheap, because the trees are layout-compatible everywhere this build - * reaches into them -- `Python.xcframework//Python.framework/Headers`, `-F .../`, - * `Python.xcframework/lib/pythonX.Y` and `...//lib-arm64/pythonX.Y` all exist in both. - * The differences are in parts nothing here reads: BeeWare adds a `platform-config/` directory - * (cross-compilation sysconfig data for building wheels) and a `VERSIONS` file, and its headers - * still carry `module.modulemap`, `lock.h`, `monitoring.h` and `typeslots.h` where 3.15 has - * `pyabi.h`, `slots.h` and `slots_generated.h` instead -- a 3.14-vs-3.15 difference, not a - * packaging one. Both ship a GIL-enabled build; neither publishes a free-threaded iOS variant. + * There is no GIL iOS build and no switch (#158; leader 2026-11-13). The python.org (3.15) and + * BeeWare (3.14 and earlier) GIL sources this build used to choose between are gone. */ -val iosFromPythonOrg = pythonVersion.split(".").let { - it[0].toInt() > 3 || (it[0].toInt() == 3 && it[1].toInt() >= 15) -} - -val iosArchiveName = if (iosFromPythonOrg) { - "python-$pythonVersion-iOS-XCframework.tar.gz" -} else { - "Python-$libVersion-iOS-support.$pythonAppleSupportBuild.tar.gz" -} -val iosUrl = if (iosFromPythonOrg) { - "https://www.python.org/ftp/python/$pythonOrgReleaseDir/$iosArchiveName" -} else { - "https://github.com/beeware/Python-Apple-support/releases/download/$libVersion-$pythonAppleSupportBuild/$iosArchiveName" -} +val iosArchiveName = "python-${configuredPythonVersion}t-iOS-XCframework.tar.gz" +val iosUrl = "$freeThreadedMobileReleaseUrl/$iosArchiveName" val iosArchive = file("$downloadDir/$iosArchiveName") val iosExtractDir = file("$extractedDir/ios") -val iosLockKey = if (iosFromPythonOrg) "ios-$pythonVersion-pythonorg" else "ios-$libVersion-$pythonAppleSupportBuild" +val iosLockKey = "ios-$configuredPythonVersion$mobileLockKeySuffix" pythonArchiveKeys[iosLockKey] = iosArchive val downloadPython_ios = tasks.register("downloadPython_ios") { inputs.property("url", iosUrl) outputs.dir(iosExtractDir) - + doLast { if (!iosArchive.exists()) { println("Downloading $iosUrl") @@ -634,24 +634,14 @@ val downloadPython_ios = tasks.register("downloadPython_ios") { } } } - - // The two iOS sources are not equally verifiable, and the difference is worth stating. - // - // BeeWare (<= 3.14) publishes five tar.gz assets and nothing else -- no checksums, no - // signatures, no GitHub attestations. The lockfile pin is the only honest instrument - // there, and no amount of build wiring changes that. - // - // python.org (3.15+) publishes a sibling `.sigstore` bundle, which IS verifiable - // here. An earlier comment claimed otherwise; it was wrong. - // - // The lockfile applies to both regardless, because it is the only check that works - // offline and the only one that covers every source. + + // The lockfile is the gate (see "What is covered" above). verifyChecksum(iosLockKey, iosArchive) - if (iosFromPythonOrg) maybeVerifySigstore(iosArchive, iosUrl) - // The iOS extract dir is keyed by pythonVersion, not by archive, so bumping the pinned - // archive (b10 -> b11) would otherwise keep the old tree: "not empty, skip". A stamp of the - // archive's lock key + digest tells a stale tree from a current one (issue #47). + // The iOS extract dir is keyed by pythonVersion, not by archive, so changing the pinned + // archive (python.org's GIL framework -> the 3.15t one) would otherwise keep the old tree: + // "not empty, skip". A stamp of the archive's lock key and name tells a stale tree from a + // current one (issue #47). val iosStamp = file("$iosExtractDir/.archive-stamp") val iosWanted = "$iosLockKey ${iosArchive.name}" if (iosExtractDir.exists() && (!iosStamp.exists() || iosStamp.readText().trim() != iosWanted)) { @@ -668,6 +658,7 @@ val downloadPython_ios = tasks.register("downloadPython_ios") { iosStamp.writeText(iosWanted + "\n") } requireHeaderVersion("ios", iosExtractDir) + requireFreeThreadedHeaders("ios", iosExtractDir) } } @@ -682,8 +673,8 @@ val downloadAllPythonBuilds by tasks.registering { * target and flavour, each carrying its extraction task. Consumers (pypackpack's compile slot, * TypedPython's C build) read it with * `project(":python-multiplatform").extensions.getByType().includeDir("macos-aarch64")` - * after `evaluationDependsOn(":python-multiplatform")`. Only the desktop flavour selected with - * `-PpythonFreeThreaded` has a task; Android and iOS are always GIL. See + * after `evaluationDependsOn(":python-multiplatform")`. Only the flavour the library embeds has a + * task, and that is free-threaded on every target: desktop (#158), Android and iOS (#328). See * `docs/platforms/python-version-acquisition.md` §7. */ extensions.add( @@ -692,9 +683,7 @@ extensions.add( layout.projectDirectory.dir(extractedDir.absolutePath), configuredPythonVersion, ) { target, flavour -> - val desktop = target in CPythonIncludeLayout.desktopTargets - val extractedFlavour = if (desktop && pythonFreeThreaded) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL - if (flavour == extractedFlavour) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null + if (flavour == CPythonIncludeLayout.embeddedFlavour(target)) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null }, ) @@ -703,13 +692,30 @@ extensions.add( * extension, as Gradle attributes on the published variants, and as a classpath resource * (`META-INF/python-multiplatform/python.properties`) in the jar and the AAR. See * `docs/platforms/python-version-acquisition.md` "Published version". + * + * Per platform since #158: the desktop jar, the AAR and every Kotlin/Native klib (macosArm64, + * linuxX64, iOS, androidNative) embed 3.15t (#158, #328); only wasm still embeds a GIL build, until + * #175. So the free-threaded attribute is decided per published variant (false on the wasm ones), + * and the jar and the AAR each get their own resource. Shared metadata variants carry no CPython and + * get no free-threaded attribute. The `pythonMultiplatform` extension holds one value, the + * free-threaded one every non-wasm platform embeds, and the per-variant attributes are the + * per-platform truth. (Extension and metadata rules: agent proposal, not confirmed.) */ -val embeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, pythonFreeThreaded) -embeddedPythonVersion.register(project) +val embeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = true) +// `android` covers the AAR's variants and androidNativeArm64/X64, `ios` iosArm64, iosX64 and +// iosSimulatorArm64. Anything else that is not metadata (the wasmJs variants) is GIL. +val freeThreadedVariantPrefixes = listOf("desktop", "macosArm64", "linuxX64", "android", "ios") +embeddedPythonVersion.register(project) { configurationName -> + when { + freeThreadedVariantPrefixes.any { configurationName.startsWith(it) } -> true + configurationName.startsWith("metadata") || "Metadata" in configurationName -> null + else -> false + } +} -val generatePythonProperties by tasks.registering { - val outputDir = layout.buildDirectory.dir("generated/python-properties") - val content = embeddedPythonVersion.renderProperties() +fun registerPythonProperties(name: String, embedded: EmbeddedPythonVersion) = tasks.register(name) { + val outputDir = layout.buildDirectory.dir("generated/python-properties/$name") + val content = embedded.renderProperties() inputs.property("content", content) outputs.dir(outputDir) doLast { @@ -719,6 +725,12 @@ val generatePythonProperties by tasks.registering { } } +/** `freeThreaded=true`: the desktop jar's resource. */ +val generateDesktopPythonProperties = registerPythonProperties("generateDesktopPythonProperties", embeddedPythonVersion) + +/** `freeThreaded=true`: the AAR's resource (3.15t since #328). */ +val generateAndroidPythonProperties = registerPythonProperties("generateAndroidPythonProperties", embeddedPythonVersion) + /** * The dev agent (SPEC L-12, #233) is Python, kept as a real file -- `src/commonMain/python/ * python_multiplatform_devagent.py`, which `src/test/python/test_devagent.py` tests with plain CPython -- and @@ -1066,7 +1078,11 @@ kotlin { } afterEvaluate { val abiList = listOf("arm64-v8a", "x86_64") - val copyAndroidPythonBinaries by tasks.creating(Copy::class) { + // A Sync, not a Copy: a Copy never removes what it stopped copying, so a jniLibs tree + // staged under an older pythonVersion (libpython3.14.so beside libpython3.15.so) kept + // shipping in the AAR. Nothing else writes this directory (the link task stages into + // `$androidBuildDir//jniLibs`), so the Sync owns all of it. + val copyAndroidPythonBinaries by tasks.creating(Sync::class) { dependsOn( tasks.named("linkAndroidNativeArm64"), tasks.named("linkAndroidNativeX64"), @@ -1075,7 +1091,7 @@ kotlin { into("$androidBuildDir/jniLibs/") abiList.forEach { abi -> val arch = if (abi == "arm64-v8a") "aarch64" else "x86_64" - from("$extractedDir/android-$arch/prefix/lib") { + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib") { include("libpython*.so") include("lib*_python.so") into(abi) @@ -1096,7 +1112,7 @@ kotlin { // The library itself is 0.49 MB of that. Two of the entries above are payload nothing // on a device ever reads, and both are dropped here: // - // - `include/python$libVersion` is CPython's C headers. cinterop reads them from the + // - `include/python$mobileTag` is CPython's C headers. cinterop reads them from the // extraction tree (`targetIncludePath`), never from assets, and no Kotlin source in // `androidMain`, `artMain`, `androidInstrumentedTest` or `sample` opens an asset // under `include/`. They were being shipped to every device for nothing. @@ -1121,10 +1137,12 @@ kotlin { doFirst { delete("$androidBuildDir/assets/") } abiList.forEach { abi -> val arch = if (abi == "arm64-v8a") "aarch64" else "x86_64" - from("$extractedDir/android-$arch/prefix/lib/python$libVersion") { - exclude("config-$libVersion-$arch-linux-android/") + // `lib/python3.15t`: the free-threaded stdlib, which is where + // `PythonBootstrap.stdlibRelativePath()` (taggedVersionString) looks (#328). + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib/python$mobileTag") { + exclude("config-$mobileTag-$arch-linux-android/") exclude("test/") - into("$abi/lib/python$libVersion") + into("$abi/lib/python$mobileTag") } } } @@ -1210,8 +1228,6 @@ kotlin { include("linux-*$desktopFlavourSuffix/python/lib/libpython*.so*") include("windows-*$desktopFlavourSuffix/python/python*.dll") include("windows-*$desktopFlavourSuffix/python/vcruntime*.dll") - // `macos-*` matches `macos-aarch64-freethreaded` too. - if (!pythonFreeThreaded) exclude("*-freethreaded/**") eachFile { if (file.length() == 0L) { exclude(); return@eachFile } val platform = relativePath.segments[0].removeSuffix(desktopFlavourSuffix) @@ -1222,20 +1238,13 @@ kotlin { } } tasks.withType().matching { it.name == "desktopJar" }.configureEach { - if (configuredPythonVersion == "3.13.0" && !pythonFreeThreaded) { - from(libPathForDesktop) { - include("windows-*/*") - include("linux-*/*") - include("macos-*/*") - into("lib") - } - } else { - // Issue #74: the libraries are packed from the staged tree, not straight from the - // extraction, so that symlinks are resolved and the linux library is stripped. - dependsOn(stageDesktopLibraries) - from(stageDesktopLibraries.map { it.destinationDir }) { - include("lib/**") - } + // Issue #74: the libraries are packed from the staged tree, not straight from the + // extraction, so that symlinks are resolved and the linux library is stripped. (A + // branch for checked-in 3.13.0 GIL binaries under `lib/desktop` used to sit here; the + // desktop build is free-threaded only since #158, so it could not be reached.) + dependsOn(stageDesktopLibraries) + from(stageDesktopLibraries.map { it.destinationDir }) { + include("lib/**") } } } @@ -1245,7 +1254,8 @@ kotlin { */ listOf( iosArm64(), iosSimulatorArm64(), iosX64(), - androidNativeArm64(), androidNativeX64() + androidNativeArm64(), androidNativeX64(), + macosArm64(), linuxX64() ).forEach { nativeTarget -> nativeTarget.apply { val targetABI = when(konanTarget) { @@ -1254,16 +1264,20 @@ kotlin { IOS_ARM64 -> "ios-arm64" IOS_X64 -> "ios-arm64_x86_64-simulator" IOS_SIMULATOR_ARM64 -> "ios-arm64_x86_64-simulator" + MACOS_ARM64 -> "macos-aarch64" + LINUX_X64 -> "linux-x86_64" else -> throw RuntimeException("Unsupported ABI: $konanTarget") } val targetExtractDir = when(konanTarget.family) { - Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}/prefix" + Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}$mobileFlavourSuffix/prefix" Family.IOS -> "$extractedDir/ios/Python.xcframework" + Family.OSX, Family.LINUX -> "$extractedDir/$targetABI$desktopFlavourSuffix/python" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } val targetIncludePath = when(konanTarget.family) { - Family.ANDROID -> "$targetExtractDir/include/python$libVersion" + Family.ANDROID -> "$targetExtractDir/include/python$mobileTag" Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers" + Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$desktopAbiSuffix" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } @@ -1284,8 +1298,11 @@ kotlin { binaries { if (konanTarget.family == Family.ANDROID) { - sharedLib("multiplatform_python$libVersion") { - linkerOpts.addAll(listOf("-L$targetExtractDir/lib/", "-lpython$libVersion", "-u", "JNI_OnLoad")) + // `libmultiplatform_python3.15t.so`: the name `manager.loadLibPython` asks for on + // Android ("multiplatform_python" + taggedVersionString), linked against the + // free-threaded `libpython3.15t.so` (#328). + sharedLib("multiplatform_python$mobileTag") { + linkerOpts.addAll(listOf("-L$targetExtractDir/lib/", "-lpython$mobileTag", "-u", "JNI_OnLoad")) linkTaskProvider.configure { val type = if (buildType == NativeBuildType.DEBUG) "debug" else "release" @@ -1295,7 +1312,14 @@ kotlin { // outputFile. That is why a changed .def or source produced an APK with // a stale library and an UnsatisfiedLinkError that looked like a code // bug -- the fix had to be built twice for the second run to pick it up. + // A library linked under an older pythonVersion + // (libmultiplatform_python3.14.so) is removed first, or it keeps + // shipping in the AAR beside the current one. doLast { + val current = outputFile.get().name + dest.listFiles { f -> + f.name.startsWith("libmultiplatform_python") && f.name.endsWith(".so") && f.name != current + }?.forEach { it.delete() } copy { from(outputFile) into(dest) @@ -1308,8 +1332,8 @@ kotlin { } } getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( - "-L$targetExtractDir/lib/", - "-lpython$libVersion", + "-L$targetExtractDir/lib/", + "-lpython$mobileTag", "-Wl,--allow-shlib-undefined", "-u", "JNI_OnLoad" )) @@ -1331,6 +1355,28 @@ kotlin { "-framework", "Python", "-F$targetExtractDir/$targetABI", "-rpath", "$targetExtractDir/$targetABI" )) + } else if (konanTarget.family == Family.OSX) { + getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( + "-L$targetExtractDir/lib", + "-lpython$libVersion$desktopAbiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + } else if (konanTarget.family == Family.LINUX) { + // python-build-standalone ships libpythonX.Y.so as a symlink to libpythonX.Y.so.1.0, + // which Gradle's tarTree extracts as a 0-byte file; stageDesktopLibraries resolves + // it into a real libpythonX.Y.so under build/desktop-libs/lib/linux-x86_64. + val stagedLinuxLibDir = layout.buildDirectory.dir("desktop-libs/lib/linux-x86_64").get().asFile.absolutePath + getTest(NativeBuildType.DEBUG).apply { + linkerOpts.addAll(listOf( + "-L$stagedLinuxLibDir", + "-L$targetExtractDir/lib", + "-lpython$libVersion$desktopAbiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + linkTaskProvider.configure { + dependsOn("stageDesktopLibraries") + } + } } } } @@ -1350,7 +1396,7 @@ kotlin { jvmTest.dependsOn(commonTest) val desktopMain by getting { resources.srcDirs("src/desktopMain/resources") - resources.srcDir(generatePythonProperties) + resources.srcDir(generateDesktopPythonProperties) // `reachability-metadata.json` is generated, not checked in -- see // GenerateReachabilityMetadata above. Wiring it as a resource srcDir (through the // task's own output provider, so the dependency is inferred) is what puts it on the @@ -1426,6 +1472,15 @@ kotlin { androidNativeX64Main.dependsOn(artMain) androidNativeArm64Main.dependsOn(artMain) + val desktopNativeMain by creating { + kotlin.srcDir("src/desktopNativeMain/kotlin") + } + val macosArm64Main by getting + val linuxX64Main by getting + desktopNativeMain.dependsOn(nativeMain) + macosArm64Main.dependsOn(desktopNativeMain) + linuxX64Main.dependsOn(desktopNativeMain) + // `src/nativeTest` existed as a directory long before this line, and nothing pointed at // it -- so its `GCLeakTest.native.kt` was dead source, and the same file had been copied // byte-for-byte into all five target test source sets to compensate. This is the mirror of @@ -1439,6 +1494,13 @@ kotlin { "androidNativeX64Test", "androidNativeArm64Test", ).forEach { getByName(it).dependsOn(nativeTest) } + val desktopNativeTest by creating { + kotlin.srcDir("src/desktopNativeTest/kotlin") + } + desktopNativeTest.dependsOn(nativeTest) + listOf("macosArm64Test", "linuxX64Test") + .forEach { getByName(it).dependsOn(desktopNativeTest) } + // ...and `artTest` completes the mirror for the other half: things true of androidNative // but not of iOS. Without it such a test has nowhere to live except duplicated into both // androidNative target source sets, which is the arrangement the comment above describes @@ -1452,22 +1514,34 @@ kotlin { } } +tasks.named("macosArm64Test") { + dependsOn("downloadPython_macos_aarch64") + val pythonHome = file("$extractedDir/macos-aarch64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + +tasks.named("linuxX64Test") { + dependsOn("downloadPython_linux_x86_64") + val pythonHome = file("$extractedDir/linux-x86_64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + /** * The iOS `Python.framework` ships only the interpreter binary and headers, it carries no * standard library. `Py_Initialize()` therefore aborts the process with * "Fatal Python error: Failed to import encodings module" unless PYTHONHOME points at a - * prefix containing `lib/python3.13`. + * prefix containing `lib/python3.15t` (the free-threaded build's stdlib directory, #328). * * The simulator distribution archive does contain that stdlib, so unpack it into the build * directory and hand its location to the test binary. */ val extractIosSimulatorStdlib by tasks.registering(Copy::class) { dependsOn(downloadAllPythonBuilds) - from("$extractedDir/ios/Python.xcframework/lib/python$libVersion") { - into("lib/python$libVersion") + from("$extractedDir/ios/Python.xcframework/lib/python$mobileTag") { + into("lib/python$mobileTag") } - from("$extractedDir/ios/Python.xcframework/ios-arm64_x86_64-simulator/lib-arm64/python$libVersion") { - into("lib/python$libVersion") + from("$extractedDir/ios/Python.xcframework/ios-arm64_x86_64-simulator/lib-arm64/python$mobileTag") { + into("lib/python$mobileTag") } into(layout.buildDirectory.dir("python-stdlib/ios-simulator")) includeEmptyDirs = false @@ -1499,8 +1573,8 @@ tasks.withType("stageAndroidNative${targetSuffix}TestPayload") { dependsOn(downloadAllPythonBuilds) - from("$extractedDir/android-$arch/prefix/lib") { + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib") { include("*.so") include("*.so.*") into("lib") } - from("$extractedDir/android-$arch/prefix/lib/python$libVersion") { - exclude("config-$libVersion-*/") - into("lib/python$libVersion") + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib/python$mobileTag") { + exclude("config-$mobileTag-*/") + into("lib/python$mobileTag") } into(stagingDir) includeEmptyDirs = false @@ -2675,12 +2749,12 @@ android { sourceSets["main"].assets.srcDirs("src/androidMain/assets", "$androidBuildDir/assets") // KMP androidMain resources do not reach the AAR's classes.jar; AGP's main resources do. - sourceSets["main"].resources.srcDir(generatePythonProperties) + sourceSets["main"].resources.srcDir(generateAndroidPythonProperties) // AGP's legacy source-set API resolves that provider to files and drops the task dependency // (issue #72: `processReleaseJavaRes` read the output with no declared dependency), so it is // declared on every AGP java-resources task explicitly. tasks.matching { Regex("""process\w*JavaRes""").matches(it.name) } - .configureEach { dependsOn(generatePythonProperties) } + .configureEach { dependsOn(generateAndroidPythonProperties) } sourceSets["androidTest"].assets.srcDirs("$androidBuildDir/assets") sourceSets["debug"].jniLibs.srcDirs("src/androidMain/jniLibs", "$androidBuildDir/jniLibs", "$androidBuildDir/debug/jniLibs") @@ -2733,47 +2807,11 @@ publishing { artifact(wasmBrowserRuntimeZip) } } - publications.withType().configureEach { - //artifact(javadocJar) - groupId = groupId - artifactId = artifactId - version = version - - pom { - name.set("python-multiplatform") - description.set("A multiplatform solution to use Python with Kotlin interoperably.") - url.set("https://github.com/thisisthepy/python-multiplatform-mobile") - licenses { - license { - name.set("The Apache License, Version 2.0") - url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") - } - } - developers { - developer { - id.set("thisisthepy") - name.set("thisisthepy") - email.set("thisisthepy@gmail.com") - } - } - scm { - connection.set("scm:git:github.com/thisisthepy/python-multiplatform-mobile.git") - developerConnection.set("scm:git:ssh://github.com/thisisthepy/python-multiplatform-mobile.git") - url.set("https://github.com/thisisthepy/python-multiplatform-mobile") - } - } - } } -//signing { -// useInMemoryPgpKeys( -// rootProject.extra["signing_key_id"] as String, -// rootProject.extra["signing_secret_key"] as String, -// rootProject.extra["signing_password"] as String -// ) -// sign(publishing.publications) -//} - +// POM metadata, the sources and javadoc jars, the centralStaging repository and the in-memory GPG +// signing (only when MAVEN_GPG_PRIVATE_KEY is set) for Maven Central: ROADMAP 15.1. +apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) fun downloadPythonBuilds() { // Moved to task `downloadAllPythonBuilds` diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt new file mode 100644 index 000000000..6fffe2e85 --- /dev/null +++ b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt @@ -0,0 +1,71 @@ +package python.multiplatform + +import python.multiplatform.env.PythonBootstrap +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.PythonTestFixture +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * SPEC T-2 (issue #328): Android (ART, arm64-v8a and x86_64) embeds the free-threaded CPython 3.15t, + * and only that build. + * + * The maintainer's decision on #158 (2026-10-04): "Free-threaded is the default and the only build. + * The GIL build is not kept as an option. Baseline is CPython 3.15t." Desktop moved first (#398); + * Android follows here, with no opt-in flag (leader, 2026-11-13). + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `androidMain`, so `abiFlags` was + * `""`, the stdlib was looked for under `lib/python3.15`, the AAR's `python.properties` said + * `freeThreaded=false`, `autoDrainInterval` defaulted to 0, and the loaded `libpython3.15.so` was + * python.org's GIL build (`Py_GIL_DISABLED` 0). Each assertion fails on that build with its own + * message. (This replaces `AndroidGilBuildTest`, which pinned the GIL build until #328.) + */ +class AndroidFreeThreadedBuildTest { + + @Test + fun androidReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "Android embeds CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theStdlibIsLookedForUnderTheFreeThreadedDirectory() { + val tagged = Versions.currentVersion.compactVersionString + "t" + assertEquals("lib/python$tagged", PythonBootstrap.stdlibRelativePath()) + assertEquals(".python-multiplatform-stdlib.$tagged", PythonBootstrap.stampFile(File("/prefix")).name) + } + + @Test + fun theAarSaysItIsFreeThreaded() { + // The AAR's `META-INF/python-multiplatform/python.properties` (generateAndroidPythonProperties). + val text = assertNotNull( + PythonBootstrap::class.java.getResourceAsStream("/META-INF/python-multiplatform/python.properties"), + "the AAR carries no META-INF/python-multiplatform/python.properties", + ).use { it.readBytes().decodeToString() } + assertTrue("freeThreaded=true" in text, "the AAR's python.properties must say freeThreaded=true (#328): $text") + } + + @Test + fun theLoadedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + // `Py_GIL_DISABLED` is a build-time fact of the libpython that was loaded; + // `sys._is_gil_enabled()` is not asserted, because an extension module without `Py_mod_gil` + // re-enables the GIL at run time on a free-threaded build. + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the Android libpython is a GIL build; Android must embed the free-threaded one (#328)", + ) + } + + @Test + fun theAutomaticCheckpointIsOnByDefault() { + assertTrue( + Python3.autoDrainInterval > 0, + "autoDrainInterval=${Python3.autoDrainInterval}: the free-threaded default turns the checkpoint on", + ) + } +} diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 02b17a19c..eb52a02d8 100644 --- a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -359,9 +359,13 @@ class CycleCollectionTest { * releases it. `pmp_proxy_dealloc` in `jni_onload.def` hand-writes that (`PyObject_Type` for * the temporary, then two `Py_DecRef`s), so it can leak one type reference per instance by * forgetting, or free the type out from under the process by doing it twice. Both are checked - * by reading `ob_refcnt` through [bindings.obRefCnt] -- the first field of `PyObject`, fixed - * there by the stable ABI. `sys.getrefcount` cannot answer this: it counts its own argument's - * temporary reference. + * by reading the type's reference count through [bindings.obRefCnt], which sums the + * free-threaded header's `ob_ref_local` and `ob_ref_shared` (Android embeds 3.15t, #328). + * `sys.getrefcount` cannot answer this: it counts its own argument's temporary reference. + * + * A heap type's references are deferred on the free-threaded build until a collection + * materialises them, so one is taken on either side of the instantiation loop, as in the + * desktop `CycleCollectionTest`. * * The "rises while alive" assertion is not decoration: it is what proves the probe reads a * real refcount, so the "returns afterwards" assertion cannot pass vacuously on an address @@ -373,6 +377,9 @@ class CycleCollectionTest { val proxyTypeAddr = ProxyTypeFactory.createProxyType() assertTrue(proxyTypeAddr != 0L, "Proxy type creation failed") + // Materialise the deferred references the type already carries (free-threaded build). + python.native.ffi.PyGC_Collect() + val liveBefore = HandleTable.liveCount val typeRefBefore = bindings.obRefCnt(proxyTypeAddr) @@ -395,6 +402,8 @@ class CycleCollectionTest { liveBefore + rounds, HandleTable.liveCount, "setup failed: $rounds handles should be rooted while the proxies are alive", ) + // The instantiations above were deferred; a collection makes them countable. + python.native.ffi.PyGC_Collect() val typeRefAlive = bindings.obRefCnt(proxyTypeAddr) assertEquals( typeRefBefore + rounds, typeRefAlive, diff --git a/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt new file mode 100644 index 000000000..7d78bf149 --- /dev/null +++ b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** Android (ART) embeds the project's own free-threaded CPython 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt index 6c9f0f29c..585848d57 100644 --- a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt +++ b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt @@ -6,7 +6,7 @@ import java.nio.ByteBuffer * Android (ART) actuals for the shape vocabulary declared in `jvmMain/.../ShapeDowncalls.kt`. * * Each shape is one JNI native-method call into a `@CName`-exported trampoline compiled into - * `libmultiplatform_python3.13.so` (see `nativeMain/.../EmbedAPI.native.kt`), which casts the + * `libmultiplatform_python3.15t.so` (see `nativeMain/.../EmbedAPI.native.kt`), which casts the * `fn` address to a typed `CFunction` pointer and invokes it directly. This is "path 1" from * `docs/design/downcall-design.md`: ordinary JNI -> trampoline -> indirect call. ART cannot synthesize * an arbitrary native call at runtime the way Kotlin/Native's cinterop can at compile time, so diff --git a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt index 1a55088e5..6bd0160bc 100644 --- a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt +++ b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt @@ -686,12 +686,14 @@ object bindings { @JvmStatic external fun proxySetHandle(proxy: Long, handle: Long) /** - * `ob_refcnt` of the object at [obj], read straight out of the header. + * `Py_REFCNT` of the object at [obj], read straight out of the header. * * `Py_REFCNT` is a macro and `sys.getrefcount` adds its argument's own temporary to the * answer, so neither can say whether `tp_dealloc` released the instance's reference to its - * heap type exactly once. `ob_refcnt` is the first field of `PyObject` and its offset is - * fixed by the stable ABI, which is the one layout fact this file is allowed to know. + * heap type exactly once. Android embeds the free-threaded 3.15t (#328), whose `PyObject` + * header splits the count into `uint32 ob_ref_local` at +12 and `Py_ssize_t ob_ref_shared` at + * +16 (shifted left by 2); offset 0 is the owning thread id. The C side sums the two, as the + * desktop `CycleCollectionTest.typeRefCount` does. * * Composed on the C side for the same reason [proxySetHandle] is: reading 8 bytes at a raw * address from Kotlin/JVM would need `sun.misc.Unsafe`, a restricted non-SDK interface here. diff --git a/python-multiplatform/src/artMain/cinterop/jni_onload.def b/python-multiplatform/src/artMain/cinterop/jni_onload.def index 92f0e76e3..892b27061 100644 --- a/python-multiplatform/src/artMain/cinterop/jni_onload.def +++ b/python-multiplatform/src/artMain/cinterop/jni_onload.def @@ -1391,13 +1391,20 @@ static void proxy_set_handle(JNIEnv *e, jclass c, jlong obj, jlong handle) { } /* - * ob_refcnt of the object at `obj`, read straight out of the header. + * Py_REFCNT of the object at `obj`, read straight out of the header. * * Py_REFCNT is a macro over a struct this file does not include, and sys.getrefcount counts its * own argument's temporary reference -- neither can answer whether tp_dealloc released the * instance's reference to its heap type exactly once, which is what the cycle-collection tests - * check. ob_refcnt is the first field of PyObject and the stable ABI fixes that; it is 64-bit on - * both ABIs this project ships (arm64-v8a, x86_64), so jlong is the right width. + * check. + * + * androidNative links the free-threaded CPython 3.15t, and only that build (#158, #328). Its + * PyObject header (Include/object.h, Py_GIL_DISABLED) is not the GIL one: offset 0 is ob_tid, the + * owning thread's id, and the count is split between a thread-local uint32 ob_ref_local at +12 and + * an atomic Py_ssize_t ob_ref_shared at +16, shifted left by _Py_REF_SHARED_SHIFT (2). Py_REFCNT + * is their sum; this reproduces it, as the desktop CycleCollectionTest's typeRefCount does. Both + * ABIs this project ships (arm64-v8a, x86_64) are 64-bit, so the offsets hold for both. The shared + * half is read with a plain load, which is the same racy poll the tests already accept. * * A test probe. Composed here for the same reason proxy_set_handle is: reading 8 bytes at a raw * address from Kotlin/JVM needs sun.misc.Unsafe, a restricted non-SDK interface on Android. @@ -1407,7 +1414,10 @@ static void proxy_set_handle(JNIEnv *e, jclass c, jlong obj, jlong handle) { */ static jlong ob_refcnt(JNIEnv *e, jclass c, jlong obj) { if (obj == 0) return 0; - return *(jlong *)(intptr_t)obj; + const char *header = (const char *)(intptr_t)obj; + uint32_t local = *(const uint32_t *)(header + 12); + int64_t shared = *(const int64_t *)(header + 16); + return (jlong)local + (jlong)(shared >> 2); } /* diff --git a/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt new file mode 100644 index 000000000..0123e05d8 --- /dev/null +++ b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** androidNative links the project's own free-threaded CPython 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt new file mode 100644 index 000000000..6e3179511 --- /dev/null +++ b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt @@ -0,0 +1,32 @@ +package python.multiplatform + +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-2 (issue #328): androidNative (`androidNativeArm64`, `androidNativeX64`) links the + * free-threaded CPython 3.15t, the same build as Android (ART). + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `artMain`, so `abiFlags` was `""`, + * and the cinterop compiled against `include/python3.15` and linked `-lpython3.15`, python.org's GIL + * build. (This replaces `AndroidNativeGilBuildTest`, which pinned the GIL build until #328.) + */ +class AndroidNativeFreeThreadedBuildTest { + + @Test + fun androidNativeReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "androidNative links CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLinkedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the androidNative libpython is a GIL build; it must be the free-threaded one (#328)", + ) + } +} diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt new file mode 100644 index 000000000..b383d0f51 --- /dev/null +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt @@ -0,0 +1,33 @@ +package python.multiplatform + + +/** + * Whether the CPython this platform embeds is a free-threaded build (PEP 703, `Py_GIL_DISABLED`). + * + * A property of the platform, not of the whole build. The maintainer's decision on issue #158 + * (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an + * option. Baseline is CPython 3.15t." The desktop half landed first (leader, 2026-11-13: "flip the + * desktop default to 3.15t now, before the #377 alpha03 release"); Android, androidNative and iOS + * followed under #328 with the project's own 3.15t builds (release `cpython-t-mobile`; leader + * 2026-11-13: no opt-in flag on mobile). wasm still embeds a GIL build until #175. So: + * + * | Source set | Targets | Embeds | + * |---------------------|---------------------------------|---------------| + * | `desktopMain` | desktop JVM | 3.15t | + * | `desktopNativeMain` | `macosArm64`, `linuxX64` | 3.15t | + * | `androidMain` | Android (ART) | 3.15t | + * | `artMain` | `androidNativeArm64`, `X64` | 3.15t | + * | `iosMain` | iOS | 3.15t | + * | `wasmJsMain` | `wasmJs` | GIL | + * + * It used to be one `BuildConfig.pythonFreeThreaded` shared by every target, which could not + * express the transition: flipping it for desktop made Android look for `lib/python3.15t`, a + * directory its GIL prefix did not have. With every non-wasm platform free-threaded it stays an + * `expect`, because wasm is still the exception (agent proposal, not confirmed: drop the + * `expect` once #175 makes wasm free-threaded too). A plain (non-`inline`) `expect val`, so the + * intermediate-source-set crash in `commonMain/README.md` does not apply. + * + * Everything CPython names after itself follows from this through [Versions.abiFlags]: the shared + * library (`libpython3.15t`), the stdlib directory (`lib/python3.15t/`), and the PYTHONHOME check. + */ +internal expect val embedsFreeThreadedPython: Boolean diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt index 49cbc440c..ef50c3d80 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt @@ -50,12 +50,13 @@ class Versions private constructor(val versionString: String) { /** * CPython's ABI flags suffix: `t` on a free-threaded build (PEP 703), empty otherwise. * - * This is a property of the *build*, not of the version -- 3.14 ships in both flavours out of - * the same source tree -- so it comes from [BuildConfig.pythonFreeThreaded] rather than from - * [versionString]. + * This is a property of the *build*, not of the version -- 3.15 ships in both flavours out of + * the same source tree -- and of the *platform*: desktop (JVM and Kotlin/Native), Android, + * androidNative and iOS embed 3.15t (issues #158, #328), wasm a GIL build until #175. So it + * comes from the per-platform [embedsFreeThreadedPython] rather than from [versionString]. */ val abiFlags: String - get() = if (BuildConfig.pythonFreeThreaded) "t" else "" + get() = if (embedsFreeThreadedPython) "t" else "" /** * `major.minor` plus [abiFlags]: `3.14` on a default build, `3.14t` free-threaded. diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index 14d153326..907972c6d 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -1,6 +1,6 @@ package python.multiplatform.ffi -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.env.PythonHomeCheck import python.multiplatform.env.PythonDevAgent import python.multiplatform.env.PythonPayload @@ -124,7 +124,9 @@ object Python3 { def _pm_install_compiled(): import builtins if not hasattr(builtins, "compiled"): - def compiled(f): + def compiled(f=None, /, **_opts): + if f is None: + return lambda g: g return f builtins.compiled = compiled _pm_install_compiled() @@ -134,51 +136,98 @@ object Python3 { } /** - * Finalize Python (Will not be able to re-initialize after this | Let this function be internal) + * Ends the interpreter (SPEC L-1a, issue #339) and returns `Py_FinalizeEx()`'s status: `0`, or + * `-1` when flushing buffered data failed. `Py_FinalizeEx()` flushes `sys.stdout` and + * `sys.stderr` itself, so output Python buffered reaches the process's streams. + * + * - Returns `0` and does nothing when the interpreter is not running. + * - Must be called on the thread that called [initialize]. From any other thread it throws + * [IllegalStateException] and the interpreter keeps running: `Py_FinalizeEx()` runs on the + * initialising thread's state, and adopting that state on another thread would leave two OS + * threads believing they own it. + * - Throws [IllegalStateException] as well when something other than [initialize] started the + * interpreter, since there is then no parked thread state to finalize on. + * - Afterwards [isInitialized] is false. Starting the interpreter again in the same process is + * not supported (extension modules and this binding's cached objects do not survive it). + * - Threads still inside [withPython] or [withGIL] are not waited for here; stop them first. */ - internal fun finalize(silent: Boolean = false) { - if (!isInitialized) return - // Py_Finalize() requires the GIL. Reclaim the state parked by initialize(). - mainThreadState?.let { - PyEval_RestoreThread(it) - mainThreadState = null + fun finalize(silent: Boolean = false): Int { + if (!isInitialized) return 0 + val parked = mainThreadState + ?: throw IllegalStateException( + "Python3.finalize(): the interpreter was not started by Python3.initialize(), so there is no thread state to finalize on" + ) + // PyGILState_GetThisThreadState needs no GIL and returns the state CPython associates with + // the calling OS thread. On the initialising thread that is the state parked by + // initialize(); on any other thread it is null or that thread's own state. + if (PyGILState_GetThisThreadState() != parked) { + throw IllegalStateException( + "Python3.finalize() must be called on the thread that called Python3.initialize()" + ) } - // Lowered BEFORE Py_Finalize(), not after. Cleaner threads consult this flag to decide + // Py_FinalizeEx() requires the GIL. Reclaim the state parked by initialize(). + PyEval_RestoreThread(parked) + mainThreadState = null + // Lowered BEFORE Py_FinalizeEx(), not after. Cleaner threads consult this flag to decide // whether releasing a reference is still legal, and finalization is exactly the window // where it stops being legal -- leaving the flag up until afterwards let a cleaner pass - // the check and call Py_DecRef while Py_Finalize() was tearing the interpreter down, + // the check and call Py_DecRef while finalization was tearing the interpreter down, // which crashed the process inside _PyObject_ClearFreeLists. // // With the flag down first, a cleaner arriving from here on returns without touching the - // C API at all. One already inside its GIL scope still holds the GIL, and Py_Finalize() + // C API at all. One already inside its GIL scope still holds the GIL, and finalization // waits for it, so that case finishes safely before teardown begins. isInitialized = false - // The checkpoint function dies with the interpreter; drop the pointer so a later - // initialize() rebuilds it instead of calling through a dangling one. Same for the - // runMain/runApp helpers, which are cached the same way and would dangle the same way. + // The checkpoint function dies with the interpreter; drop the pointer so nothing calls + // through a dangling one. Same for the runMain/runApp helpers, which are cached the same way. checkpointCallable = null builtinCompiledInstalled = false runModuleCallable = null runTopLevelCallable = null runFileCallable = null - memScoped { - Py_Finalize() - // No error message is printable here, and that is a property of the C API rather than - // an omission. `Py_Finalize()` returns void; by the time it returns there is no - // interpreter left to hold an error indicator, so `PyErr_*` cannot be consulted. The - // only status available at all is `Py_FinalizeEx()`'s `int` -- 0, or -1 when flushing - // buffered data failed -- which carries no message either. Switching to it is a real - // (small) improvement and is recorded in ROADMAP §12 rather than done here, because - // finalization is untested: the only caller is `artMain/JniExport.kt`, and a test - // that exercises it destroys the interpreter the rest of the suite shares. - if (Py_IsInitialized() != 0) { - isInitialized = true - throw IllegalStateException("Python finalization failed") + // No error message is available after this, and that is a property of the C API: once + // Py_FinalizeEx() returns there is no interpreter left to hold an error indicator. Its int + // status is the whole report. + val status = Py_FinalizeEx() + if (Py_IsInitialized() != 0) { + isInitialized = true + throw IllegalStateException("Python finalization failed") + } + if (!silent) println("INFO: Python finalized (status $status)") + return status + } + + /** + * Flushes `sys.stdout` and `sys.stderr` (SPEC L-1a, issue #339). Python buffers its standard + * streams, and a process that exits without [finalize] loses what is still in the buffer. A + * stream that is `None` or has no `flush` is skipped; an exception raised by `flush` surfaces + * as a [PyException]. + * + * Runs in a fresh dictionary, so it leaves no names behind in `__main__`. + */ + fun flush() { + withPython { + // PyDict_New: new reference, released below. + val globals = PyDict_New() ?: throw pyErrorOrGeneric("Failed to allocate the flush namespace") + try { + // PyRun_String: new reference (None) on success, null + exception set on failure. + val result = PyRun_String(FLUSH_SOURCE, PY_FILE_INPUT, globals, globals) + ?: throw pyErrorOrGeneric("Flushing Python's standard streams failed") + Py_DecRef(result) + } finally { + Py_DecRef(globals) } - if (!silent) println("INFO: Python finalized successfully!") } } + private const val FLUSH_SOURCE = """ +import sys +for _s in (sys.stdout, sys.stderr): + _f = getattr(_s, "flush", None) + if _f is not None: + _f() +""" + /** * Runs the module [moduleName] as `__main__`, the way `python -m ` does, in an * interpreter that is **still alive when this returns**. @@ -723,6 +772,9 @@ def __pmp_run_file__(path, argv): /** `Py_file_input`, the compiler-mode token for a sequence of statements (as opposed to a single expression). */ private const val PY_FILE_INPUT: Int = 257 + /** `Py_eval_input`, the start symbol of the one-argument [eval]. */ + private const val PY_EVAL_INPUT: Int = 258 + /** * Outermost [withGIL] scopes between automatic checkpoints when they are enabled. * @@ -807,6 +859,10 @@ def __pmp_run_file__(path, argv): * to take none automatically. A checkpoint is additionally skipped when nothing has been * released since the previous one, so a workload that never drops a wrapper never pays. * + * Which build that is depends on the platform ([python.multiplatform.embedsFreeThreadedPython]): + * desktop, Android, androidNative and iOS embed 3.15t (issues #158, #328), so this defaults to + * on there; wasm embeds a GIL build until #175, so it defaults to off there. + * * Defaults to on for free-threaded builds, where deferred release is the defect described on * [drainPendingReleases], a build with the global lock frees a *plain* reference the * cleaner's `Py_DecRef` immediately, with no queue to drain, so that half of the checkpoint's @@ -825,7 +881,7 @@ def __pmp_run_file__(path, argv): * call [PyGC_Collect] directly for an immediate, unconditional collection, [drainPendingReleases] * and `PyGC_Collect` both work regardless of this setting. */ - var autoDrainInterval: Int = if (BuildConfig.pythonFreeThreaded) DEFAULT_AUTO_DRAIN_INTERVAL else 0 + var autoDrainInterval: Int = if (embedsFreeThreadedPython) DEFAULT_AUTO_DRAIN_INTERVAL else 0 /** Diagnostic counters for the checkpoint machinery. Test-visible. */ internal object CheckpointCounter { @@ -944,6 +1000,35 @@ def __pmp_run_file__(path, argv): } } + /** + * Evaluates one expression in `__main__`'s namespace and returns the result as a new + * [PyObject] (SPEC L-1a, issue #339). The common case of the four-argument form, with + * `Py_eval_input`: a statement is a `SyntaxError`, raised as a [PyException]. Use [exec] for + * statements. + */ + fun eval(expression: String): PyObject { + return withPython { + // PyImport_AddModuleRef: new/strong reference to the __main__ module. + val modulePointer = PyImport_AddModuleRef("__main__") + ?: throw pyErrorOrGeneric("Failed to access the __main__ module") + try { + // PyObject_GetAttrString: new reference to __main__'s globals dict. + val globalsPointer = PyObject_GetAttrString(modulePointer, "__dict__") + ?: throw pyErrorOrGeneric("Failed to access __main__.__dict__") + try { + // PyRun_String: new reference on success, adopted by the wrapper. + val result = PyRun_String(expression, PY_EVAL_INPUT, globalsPointer, globalsPointer) + ?: throw pyErrorOrGeneric("Python eval failed") + PyObject(result, false) + } finally { + Py_DecRef(globalsPointer) + } + } finally { + Py_DecRef(modulePointer) + } + } + } + /** * Evaluate Python script */ diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt index 25aa786c9..2281a5327 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt @@ -2293,7 +2293,7 @@ object PythonxAdapter { def __dir__(): names = set(_BY_PACKAGE.get(kotlin_package, {})) names.update(_CHILDREN.get(kotlin_package, ())) - for served in (_BY_RECEIVER, _METHODS): + for served in (_BY_RECEIVER, _METHODS, _PROPERTIES): for type_name in served: package, _, leaf = type_name.rpartition('.') if package == kotlin_package: diff --git a/python-multiplatform/src/commonTest/README.md b/python-multiplatform/src/commonTest/README.md index 60bcf485b..b7e031387 100644 --- a/python-multiplatform/src/commonTest/README.md +++ b/python-multiplatform/src/commonTest/README.md @@ -13,8 +13,8 @@ platform-only file actually needs to be there, and one promotion that was tried | Execution path | Gradle task | Source-set chain | |---|---|---| -| desktop | `:python-multiplatform:desktopTest` | `desktopTest -> jvmTest -> commonTest` | -| desktop, free-threaded | same task, `-PpythonFreeThreaded=true` | same chain, linked against `libpython3.14t` instead of `libpython3.14` | +| desktop | `:python-multiplatform:desktopTest` | `desktopTest -> jvmTest -> commonTest`, against the free-threaded `libpython3.15t` (the only desktop build since #158) | +| desktop native | `:python-multiplatform:macosArm64Test`, `linuxX64Test` | `{macosArm64Test, linuxX64Test} -> desktopNativeTest -> nativeTest -> commonTest`, linked against `libpython3.15t` | | iOS simulator | `:python-multiplatform:iosSimulatorArm64Test` | `iosSimulatorArm64Test -> nativeTest -> commonTest` | | androidNative | `:python-multiplatform:androidNative{Arm64,X64}Test` (custom `adb`-driving task; `compileTestKotlinAndroidNativeArm64` for a device-free check) | `androidNative*Test -> artTest -> nativeTest -> commonTest` | | Android (ART) | `connectedAndroidTest` (instrumented, needs a device/emulator) | `androidInstrumentedTest -> jvmTest -> commonTest` | diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt index c64073c1b..fc49790de 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt @@ -13,15 +13,20 @@ import kotlin.test.assertTrue * therefore does not degrade gracefully: `manager.loadLibPython` throws `UnsatisfiedLinkError` for * a file that was downloaded and is sitting right there under a different name. * - * These assertions are written against [BuildConfig.pythonFreeThreaded] rather than hard-coding one - * flavour, so the same suite is meaningful under `-PpythonFreeThreaded=true`. + * These assertions are written against the per-platform [embedsFreeThreadedPython] rather than + * hard-coding one flavour, because the flavour differs by platform: desktop, Android, + * androidNative and iOS embed 3.15t (issues #158, #328), wasm a GIL build until #175. The platform + * test source sets pin the literal value for their platform (`DesktopFreeThreadedBuildTest`, + * `DesktopNativeFreeThreadedBuildTest`, `AndroidFreeThreadedBuildTest`, + * `AndroidNativeFreeThreadedBuildTest`, `IosFreeThreadedBuildTest`, `WasmGilBuildTest`), which is + * what a wrong actual would fail. */ class VersionsTest { @Test fun abiFlagsFollowsTheConfiguredFlavour() { assertEquals( - if (BuildConfig.pythonFreeThreaded) "t" else "", + if (embedsFreeThreadedPython) "t" else "", Versions.currentVersion.abiFlags, "abiFlags must be CPython's flags suffix: 't' free-threaded (PEP 703), empty otherwise" ) @@ -31,7 +36,7 @@ class VersionsTest { fun taggedVersionStringNamesTheArtefactsOnDisk() { val v = Versions.currentVersion assertEquals(v.compactVersionString + v.abiFlags, v.taggedVersionString) - if (BuildConfig.pythonFreeThreaded) { + if (embedsFreeThreadedPython) { assertTrue( v.taggedVersionString.endsWith("t"), "a free-threaded build must ask for libpython${v.compactVersionString}t, " + diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt index 5d3ef407f..e7765283f 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt @@ -11,7 +11,9 @@ class BuiltinCompiledTest { @Test fun compiledResolvesWithoutImportAndIsIdentity() = PythonTestFixture.withInterpreter { assertEquals("True", evalStr("compiled(len) is len")) - Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f") + assertEquals("True", evalStr("compiled(parallel=True)(len) is len")) + assertEquals("True", evalStr("compiled()(len) is len")) + Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f and compiled(parallel=True)(_bc_f) is _bc_f") assertEquals("True", evalStr("_bc_r")) // It must live in builtins, not leak into __main__. assertEquals("False", evalStr("'compiled' in globals()")) diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt index 480ae1dee..d8f7a4a80 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt @@ -58,6 +58,56 @@ class Python3Test { } } + // SPEC L-1a (#339): the one-argument eval. Red before #339 as a compile error (no such + // overload), which cannot be mistaken for a behavioural regression. + + @Test + fun evalWithOneArgumentReadsMainNamespace() = PythonTestFixture.withInterpreter { + Python3.exec("x_for_one_arg_eval = 20") + val result = Python3.eval("x_for_one_arg_eval + 22") + assertEquals("42", result.toString()) + } + + @Test + fun evalWithOneArgumentRejectsAStatement() = PythonTestFixture.withInterpreter { + // Py_eval_input, not Py_file_input: an assignment is a SyntaxError, not a silent None. + assertFailsWith { Python3.eval("y_for_one_arg_eval = 1") } + } + + // SPEC L-1a (#339): flush(). The end-to-end proof that buffered output reaches the process's + // stdout is desktopTest's InterpreterShutdownTest; these pin the edge cases in-process. + + @Test + fun flushSkipsAMissingStream() = PythonTestFixture.withInterpreter { + Python3.exec("import sys as _pm_t_sys; _pm_t_saved = _pm_t_sys.stdout; _pm_t_sys.stdout = None") + try { + Python3.flush() + } finally { + Python3.exec("_pm_t_sys.stdout = _pm_t_saved; del _pm_t_saved, _pm_t_sys") + } + } + + @Test + fun flushReportsAFailingStreamAsPyException() = PythonTestFixture.withInterpreter { + Python3.exec( + """ + import sys as _pm_t_sys + class _PmFailingStream: + def write(self, s): return len(s) + def flush(self): raise ValueError("flush failed on purpose") + _pm_t_saved = _pm_t_sys.stderr + _pm_t_sys.stderr = _PmFailingStream() + """.trimIndent() + ) + try { + val error = assertFailsWith { Python3.flush() } + // The real ValueError, not a generic failure. + assertTrue("flush failed on purpose" in error.value.toString(), "got: ${error.value} / ${error.message}") + } finally { + Python3.exec("_pm_t_sys.stderr = _pm_t_saved; del _pm_t_saved, _pm_t_sys, _PmFailingStream") + } + } + @Test fun importReturnsAUsableModule() = PythonTestFixture.withInterpreter { val math = Python3.import("math") diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt index 8140363d7..073deeefe 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt @@ -144,7 +144,12 @@ class PythonxCallableTest { _px_base = sys.getrefcount(_px_body) """.trimIndent(), ) - val base = eval("_px_base").toInt() + // Long, not Int. On the free-threaded build a module-level function is created with + // deferred reference counting (`func_new`: `_PyObject_SetDeferredRefcount` when the code is + // not nested), and `sys.getrefcount` reports the deferred bias with it: 2**60 + 1 on + // 3.15t, measured. That does not fit an Int. The bias is a constant, so every delta below + // still measures exactly the references the Kotlin side took and gave back. + val base = eval("_px_base").toLong() PythonCallables.withScope(scope) { pushComposer() @@ -160,19 +165,19 @@ class PythonxCallableTest { popComposer() } } - val held = eval("_px_held").toInt() + val held = eval("_px_held").toLong() assertTrue(held > base, "nothing on the Kotlin side took a reference: $base -> $held") assertEquals(1, scope.liveCount, "the scope did not record the callable it created") assertEquals(1, scope.close(), "closing the scope released nothing") Python3.exec("_px_released = sys.getrefcount(_px_body)") - assertEquals(base, eval("_px_released").toInt(), "the scope did not give the reference back") + assertEquals(base, eval("_px_released").toLong(), "the scope did not give the reference back") // The half a leak test does not cover. A second close must be a no-op, not a second decref: // one reference over-released here is a use-after-free somewhere unrelated later. assertEquals(0, scope.close(), "closing twice released something twice") Python3.exec("_px_twice = sys.getrefcount(_px_body)") - assertEquals(base, eval("_px_twice").toInt(), "a second close dropped the reference again") + assertEquals(base, eval("_px_twice").toLong(), "a second close dropped the reference again") } /** diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt index 3ef23f66a..a0f6c8013 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.ReleaseCounter @@ -21,10 +21,13 @@ import kotlin.time.measureTime * and friends therefore never merges the free-threaded build's biased reference-counting queue, * never processes QSBR-deferred frees and never runs a scheduled cyclic collection. * - * The consequence measured in §9 is that references handed back by the cleaner *stay* handed back - * and never freed: the counts are correct, the memory is not returned. These tests pin the shape - * of that, the fact that the obvious Stable ABI candidate does not fix it, and the cost of the one - * that does. + * The consequence measured in §9 on 3.14t is that references handed back by the cleaner *stay* + * handed back and never freed: the counts are correct, the memory is not returned. CPython 3.15 + * narrowed that: `_Py_brc_queue_object` (`Python/brc.c:118` at v3.15.0) now merges the owner's + * whole queue on its behalf when the owning thread is *detached*, which a Kotlin thread is between + * two `withGIL` scopes. What is left for the checkpoint is what was queued while the owner was + * attached. These tests pin the shape of that, the fact that the obvious Stable ABI candidate does + * not merge, and the cost of the one that does. */ class EvalCheckpointTest { @@ -125,8 +128,28 @@ class EvalCheckpointTest { * * where a reading exists that belongs to this test alone, the loop waits on that instead of * on the counter. With the global lock a released outer wrapper frees its list and hands the * target back a reference, so the target's own count says how many of *ours* the cleaner has - * reached. The free-threaded build has no such reading -- that it has none is the very thing - * the branch below asserts -- and there the drained counter is the evidence. + * reached. Free-threaded 3.15 has that reading too for every release that arrived while this + * thread was detached, but not for one that arrived while it was attached: that one sits in + * this thread's queue until a checkpoint, and the count never reaches the baseline on its + * own. So there the loop also stops once the cleaner has gone quiet, and the drain below is + * what has to finish the job. + * + * ### What the free-threaded branch asserts, and why it changed with 3.15 + * + * On 3.14t this branch asserted that the cleaner's $OUTER decrefs left the target's count + * exactly where it was: every one was queued to the owning thread. 3.15.0 added the detached + * owner case to `_Py_brc_queue_object` (`Python/brc.c:118`, `_PyThreadState_TrySuspendDetached`): + * if the owner is detached, the queuing thread suspends it, merges its whole queue and frees + * what reached zero. That is the state this thread is in while [collectorTest] waits, so the + * outer lists are now freed by the cleaner itself. What still holds, and is asserted: + * + * * the count dropped before any checkpoint was taken (the 3.15 merge exists; a return to the + * 3.14 queue-only behaviour reads `after the cleaner == after the build` and fails here); + * * `Py_MakePendingCalls`, inside one attached window so that no on-behalf merge can happen + * during it, does not move the count; + * * after [Python3.drainPendingReleases] every one of the $OUTER lists is gone, the same + * `<= refBefore` bound the global-lock branch is held to, which a drain that merged only + * part of the queue cannot pass. */ @Test fun testDrainPendingReleasesReclaimsWhatTheCleanerGaveBack(): CollectorTestResult { @@ -179,6 +202,11 @@ class EvalCheckpointTest { var attempts = 0 var cleanerReleases = 0 var refNow = refAfterBuild + // Free-threaded only: consecutive attempts in which the cleaner released nothing of + // its own, so that a straggler parked in this thread's queue cannot hold the loop open + // until the budget runs out. + var lastCleanerReleases = -1 + var cleanerQuietTurns = 0 // Driven rather than spun. On wasmJs the cleaner is a host finalisation callback that // arrives on a task, so a `while` loop here would ask 200 times inside one job and see @@ -207,10 +235,15 @@ class EvalCheckpointTest { refNow = refCount(target) attempts++ cleanerReleases = ReleaseCounter.released - releasedBefore - attempts * releasesPerProbe - // With the global lock, `refNow` is the reading that belongs to this test and - // to nothing else, so it is not enough for the counter alone to be satisfied. + if (cleanerReleases == lastCleanerReleases) cleanerQuietTurns++ + else { cleanerQuietTurns = 0; lastCleanerReleases = cleanerReleases } + // `refNow` is the reading that belongs to this test and to nothing else, so it + // is not enough for the counter alone to be satisfied. Free-threaded, a release + // that arrived while this thread was attached is queued, not freed, so a + // cleaner that has gone quiet ends the wait as well (see the KDoc). cleanerReleases >= OUTER && - (BuildConfig.pythonFreeThreaded || refNow <= refBefore) + (refNow <= refBefore || + (embedsFreeThreadedPython && cleanerQuietTurns >= QUIET_TURNS_TO_SETTLE)) } }, finish = { try { @@ -234,25 +267,34 @@ class EvalCheckpointTest { val refAfterCleaner = refCount(target) - if (BuildConfig.pythonFreeThreaded) { - assertEquals( - refAfterBuild, - refAfterCleaner, - "free-threaded: a Py_DecRef from a thread that does not own the object is " + - "queued to its owner rather than run, so $OUTER completed decrefs must " + - "leave the count exactly where it was. If this now differs, the runtime " + - "has started merging somewhere else and the rest of this test is stale." + if (embedsFreeThreadedPython) { + // 3.15's on-behalf merge (Python/brc.c:118): this thread was detached for + // almost all of the wait, so the cleaner itself freed the outer lists. + assertTrue( + refAfterCleaner < refAfterBuild, + "free-threaded 3.15: a Py_DecRef from another thread while the owner is " + + "detached merges the owner's queue on its behalf, so the cleaner " + + "alone should have freed outer lists and lowered the count. It did " + + "not move (before the build: $refBefore, after it: $refAfterBuild, " + + "after the cleaner: $refAfterCleaner; $diagnosis): the runtime is back " + + "to queuing every cross-thread release for the owner's next checkpoint" ) - // Py_MakePendingCalls is the Stable ABI function that looks like it should do - // this. It does not: it handles _PY_CALLS_TO_DO_BIT and _PY_SIGNALS_PENDING_BIT - // and returns, while the merge hangs off _PY_EVAL_EXPLICIT_MERGE_BIT, which only - // _Py_HandlePending clears. - val pendingResult = withGIL { Py_MakePendingCalls() } + // Py_MakePendingCalls is the Stable ABI function that looks like it should + // merge. It does not: it handles _PY_CALLS_TO_DO_BIT and + // _PY_SIGNALS_PENDING_BIT and returns, while the merge hangs off + // _PY_EVAL_EXPLICIT_MERGE_BIT, which only _Py_HandlePending clears. All + // three steps run in one attached window: while this thread is attached no + // other thread can merge its queue for it, so a change here is the call's. + val (beforePending, pendingResult, afterPending) = withGIL { + val a = refCount(target) + val r = Py_MakePendingCalls() + Triple(a, r, refCount(target)) + } assertEquals(0, pendingResult, "Py_MakePendingCalls should not have failed") assertEquals( - refAfterBuild, - refCount(target), + beforePending, + afterPending, "Py_MakePendingCalls must not be mistaken for a checkpoint: it does not merge " + "the biased reference-counting queue" ) @@ -260,9 +302,12 @@ class EvalCheckpointTest { Python3.drainPendingReleases() val refAfterDrain = refCount(target) assertTrue( - refAfterDrain < refAfterBuild, - "reaching an eval-loop checkpoint should have merged all $OUTER queued " + - "releases (before: $refAfterBuild, after: $refAfterDrain)" + refAfterDrain <= refBefore, + "after an eval-loop checkpoint every one of the $OUTER outer lists should " + + "be gone, the ones the cleaner freed and the ones queued while this " + + "thread was attached (before the build: $refBefore, after it: " + + "$refAfterBuild, after the cleaner: $refAfterCleaner, after the " + + "drain: $refAfterDrain; $diagnosis)" ) } else { // `<= refBefore`, not merely `< refAfterBuild`: every one of the $OUTER @@ -319,7 +364,7 @@ class EvalCheckpointTest { val gcCollectNs = timeNs(100) { withGIL { PyGC_Collect() } } println( - "\n--- eval-loop checkpoint cost (freeThreaded=${BuildConfig.pythonFreeThreaded}) ---\n" + + "\n--- eval-loop checkpoint cost (freeThreaded=${embedsFreeThreadedPython}) ---\n" + "withGIL { } ${emptyScopeNs.format()} ns/op\n" + "Python3.drainPendingReleases() ${drainNs.format()} ns/op\n" + "withGIL { Py_MakePendingCalls } ${makePendingNs.format()} ns/op (does NOT merge)\n" + diff --git a/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt b/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt new file mode 100644 index 000000000..ad134981a --- /dev/null +++ b/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** Desktop JVM embeds python-build-standalone's free-threaded CPython (issue #158). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt new file mode 100644 index 000000000..7c5453452 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** macosArm64 and linuxX64 link python-build-standalone's free-threaded CPython (issue #158). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt new file mode 100644 index 000000000..ec3d202f4 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt @@ -0,0 +1,100 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import platform.posix.closedir +import platform.posix.opendir + +/** + * The directory containing the running executable, or null when the OS cannot report it. + */ +internal expect fun executableDirectory(): String? + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) finds its packaged CPython + * prefix (SPEC L-9 / P-1, #191). + * + * A desktop native binary has no JVM, no system properties and no launcher to set `PYTHONHOME`: + * like a GraalVM native image (#246), its prefix ships beside the executable under + * [DIRECTORY_NAME] (or, inside a macOS `.app` bundle, under `../Resources/`[DIRECTORY_NAME]). + */ +internal object DesktopNativePythonHome { + + /** Matches `PackagedPythonHome.DIRECTORY_NAME` (`jvmMain`) and `IosPythonHome.DIRECTORY_NAME` (`iosMain`). */ + const val DIRECTORY_NAME = "python-multiplatform-home" + + /** A resolved prefix and a human-readable account of where it came from, for error messages. */ + data class Resolved(val path: String, val source: String) + + /** [resolve] against this process's environment and executable directory. */ + fun resolve(): Resolved? = resolve( + environmentHome = readEnvVar("PYTHONHOME"), + executableDir = executableDirectory(), + isDirectory = ::isDirectory, + ) + + /** + * The packaged prefix, or null when there is none to apply. + * + * 1. A non-blank `PYTHONHOME` in the environment wins, and the answer is null: CPython reads it + * itself and [PythonHomeCheck] already checked it. This keeps the Gradle host test tasks + * (`macosArm64Test`, `linuxX64Test`) and an explicit caller environment working unchanged. + * 2. `/`[DIRECTORY_NAME], when that directory exists. + * 3. `/../Resources/`[DIRECTORY_NAME], when that directory exists (a macOS + * `.app` bundle places the binary in `Contents/MacOS/` and resources in `Contents/Resources/`). + */ + fun resolve( + environmentHome: String?, + executableDir: String?, + isDirectory: (String) -> Boolean, + ): Resolved? { + if (!environmentHome.isNullOrBlank()) return null + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return null + val beside = "$dir/$DIRECTORY_NAME" + if (isDirectory(beside)) { + return Resolved(beside, "$DIRECTORY_NAME beside the executable ($dir)") + } + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/$DIRECTORY_NAME" + if (isDirectory(resources)) { + return Resolved(resources, "$DIRECTORY_NAME in the app bundle's Resources directory ($parent/Resources)") + } + } + return null + } + + /** + * A directory exists at [path] (a regular file there is not one): `opendir(3)` succeeds only on + * a directory. Not `stat(2)`: `st_mode` is 16 bits on macOS and 32 on Linux, and the metadata + * compilation of this shared source set, which publishing runs, rejects that mismatch. + */ + @OptIn(ExperimentalForeignApi::class) + internal fun isDirectory(path: String): Boolean { + val dir = opendir(path) ?: return false + closedir(dir) + return true + } +} + +/** + * Hands a packaged prefix to CPython with `Py_SetPythonHome` before `Py_Initialize()`. + * + * `Py_DecodeLocale` is documented by CPython as safe to call before initialisation and produces + * storage that lives for the duration of the process, as `Py_SetPythonHome` requires. + */ +@OptIn(ExperimentalForeignApi::class) +@Suppress("DEPRECATION") +internal actual fun applyPackagedPythonHome() { + val home = DesktopNativePythonHome.resolve() ?: return + try { + PythonHomeCheck.verifyOrThrow(home.path) + } catch (e: IllegalStateException) { + throw IllegalStateException( + "The Python home this application was packaged with (${home.source}) is unusable: ${e.message}", + e, + ) + } + val wide = python.native.ffi.bindings.Py_DecodeLocale(home.path, null) + ?: throw IllegalStateException("Py_DecodeLocale could not decode the packaged Python home '${home.path}'.") + python.native.ffi.bindings.Py_SetPythonHome(wide) +} diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt new file mode 100644 index 000000000..cb3c3d1e0 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt @@ -0,0 +1,24 @@ +package python.multiplatform.env + +/** + * Desktop Kotlin/Native's answer to "where did the payload land": `python/` beside the executable + * or, in a macOS `.app` bundle, under `../Resources/python`. + */ +internal actual fun discoverStagedPayloadRoots(): List = + discoverStagedPayloadRoots(executableDirectory(), ::pathIsAccessible) + +internal fun discoverStagedPayloadRoots( + executableDir: String?, + isAccessible: (String) -> Boolean, +): List { + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return emptyList() + val roots = mutableListOf() + val beside = "$dir/${PythonPayload.PAYLOAD_ROOT}" + if (isAccessible(beside)) roots += beside + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/${PythonPayload.PAYLOAD_ROOT}" + if (resources != beside && isAccessible(resources)) roots += resources + } + return roots +} diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt new file mode 100644 index 000000000..b96a99d3c --- /dev/null +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt @@ -0,0 +1,23 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-1 (issue #158): desktop Kotlin/Native (`macosArm64`, `linuxX64`) links the free-threaded + * CPython 3.15t, the same build as the desktop JVM. + * + * Red before the change: `abiFlags` came from the one `BuildConfig.pythonFreeThreaded` (default + * false), so it was `""` here, and the cinterop compiled against `include/python3.15` and linked + * `-lpython3.15`. This source set shares `nativeMain` with iOS and androidNative, which embed 3.15t + * as well since #328 (`IosFreeThreadedBuildTest`, `AndroidNativeFreeThreadedBuildTest`). + */ +class DesktopNativeFreeThreadedBuildTest { + + @Test + fun desktopNativeReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "desktop Kotlin/Native links CPython ${v.compactVersionString}t (#158)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } +} diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt new file mode 100644 index 000000000..28fb45434 --- /dev/null +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt @@ -0,0 +1,93 @@ +package python.multiplatform.env + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) resolves its packaged + * CPython prefix and payload from. + */ +class DesktopNativePythonHomeTest { + + private val besideBinary = "/apps/demo/python-multiplatform-home" + private val inAppBundle = "/apps/Demo.app/Contents/Resources/python-multiplatform-home" + + @Test + fun theDirectoryNameMatchesThePluginAndDesktopAndIos() { + assertEquals("python-multiplatform-home", DesktopNativePythonHome.DIRECTORY_NAME) + } + + @Test + fun anEnvironmentPythonHomeWinsAndNothingIsApplied() { + assertNull( + DesktopNativePythonHome.resolve( + environmentHome = "/from/env", + executableDir = "/apps/demo", + isDirectory = { true }, + ) + ) + } + + @Test + fun thePrefixBesideTheExecutableIsUsedWhenPresent() { + for (environment in listOf(null, "", " ")) { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = environment, + executableDir = "/apps/demo", + isDirectory = { it == besideBinary }, + ) + assertEquals(besideBinary, resolved?.path, "PYTHONHOME='$environment'") + assertTrue("beside the executable" in resolved!!.source, resolved.source) + } + } + + @Test + fun anAppBundleResourceDirectoryIsCheckedWhenNothingSitsBesideTheBinary() { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = null, + executableDir = "/apps/Demo.app/Contents/MacOS", + isDirectory = { it == inAppBundle }, + ) + assertEquals(inAppBundle, resolved?.path) + assertTrue("Resources" in resolved!!.source, resolved.source) + } + + @Test + fun anUnpackagedExecutableBehavesAsBefore() { + assertNull(DesktopNativePythonHome.resolve(null, "/apps/demo") { false }) + assertNull(DesktopNativePythonHome.resolve(null, null) { true }) + assertNull(DesktopNativePythonHome.resolve(null, "") { true }) + } + + @Test + fun theHostTestTaskPassesPythonHomeAndExecutableDirectoryResolves() { + val home = assertNotNull(readEnvVar("PYTHONHOME"), "the host test task must pass PYTHONHOME") + assertNull(DesktopNativePythonHome.resolve()) + val exeDir = assertNotNull(executableDirectory(), "executableDirectory() returned null") + assertTrue(DesktopNativePythonHome.isDirectory(exeDir), exeDir) + assertTrue(DesktopNativePythonHome.isDirectory(home), home) + assertFalse(DesktopNativePythonHome.isDirectory("$home/does-not-exist")) + } + + @Test + fun payloadDiscoveryFindsBesideExecutableAndBundleResources() { + assertEquals( + listOf("/apps/demo/python"), + discoverStagedPayloadRoots("/apps/demo") { it == "/apps/demo/python" }, + ) + assertEquals( + listOf("/apps/Demo.app/Contents/Resources/python"), + discoverStagedPayloadRoots("/apps/Demo.app/Contents/MacOS") { + it == "/apps/Demo.app/Contents/Resources/python" + }, + ) + assertEquals( + emptyList(), + discoverStagedPayloadRoots("/apps/demo") { false }, + ) + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt new file mode 100644 index 000000000..f8533abaf --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt @@ -0,0 +1,56 @@ +package python.multiplatform + +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * SPEC T-1 (issue #158): desktop embeds the free-threaded CPython 3.15t, and only that build. + * + * The maintainer's decision on #158 (2026-10-04): "Free-threaded is the default and the only build. + * The GIL build is not kept as an option. Baseline is CPython 3.15t." Desktop moved first (leader, + * 2026-11-13); Android and iOS follow under #328. + * + * Red before the change: the desktop build was GIL unless `-PpythonFreeThreaded=true` was passed, + * so `abiFlags` was `""`, the tagged version was `3.15`, the interpreter reported + * `Py_GIL_DISABLED = 0` and `autoDrainInterval` defaulted to 0. Each assertion below fails on that + * build with its own message. + * + * What this cannot find: what Android, androidNative, iOS and wasm embed. Their literal values are + * pinned in their own test source sets (`AndroidFreeThreadedBuildTest`, + * `AndroidNativeFreeThreadedBuildTest`, `IosFreeThreadedBuildTest`, `WasmGilBuildTest`). + */ +class DesktopFreeThreadedBuildTest { + + @Test + fun desktopReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "desktop embeds CPython ${v.compactVersionString}t (#158)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLoadedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + // The claim above is only worth something if the libpython that `manager.loadLibPython` + // actually loaded agrees with it. `Py_GIL_DISABLED` is a build-time fact of that library; + // `sys._is_gil_enabled()` is not asserted, because an extension module without + // `Py_mod_gil` re-enables the GIL at run time on a free-threaded build. + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the desktop libpython is a GIL build; desktop must embed the free-threaded one (#158)", + ) + } + + @Test + fun theAutomaticCheckpointIsOnByDefault() { + // `Python3.autoDrainInterval` is a mutable default that other tests save and restore; + // a positive value is what the free-threaded default means (Python3.kt). + assertTrue( + Python3.autoDrainInterval > 0, + "autoDrainInterval=${Python3.autoDrainInterval}: the free-threaded default turns the checkpoint on", + ) + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt index 9ce815401..eeac2a85f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt @@ -5,7 +5,17 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -/** Issue #61: the published jar carries the CPython version it embeds, agreeing with BuildConfig. */ +/** + * Issue #61: the published jar carries the CPython version it embeds, agreeing with BuildConfig. + * + * Issue #158: the desktop jar embeds the free-threaded 3.15t, so its resource says + * `freeThreaded=true`. The AAR carries its own resource, generated separately in + * `python-multiplatform/build.gradle.kts`; it says `freeThreaded=true` too since #328 + * (`AndroidFreeThreadedBuildTest.theAarSaysItIsFreeThreaded`). + * + * Red before #158: one resource served the jar and the AAR, rendered from + * `-PpythonFreeThreaded` (default false), so the desktop jar said `freeThreaded=false`. + */ class PublishedPythonVersionResourceTest { // Same path as resourcePath; that class lives in the build plugin and is // not on the test classpath. @@ -18,6 +28,6 @@ class PublishedPythonVersionResourceTest { val props = stream.use { it.readBytes().decodeToString() }.lines() .filter { '=' in it }.associate { it.substringBefore('=').trim() to it.substringAfter('=').trim() } assertEquals(BuildConfig.pythonVersion, props["pythonVersion"]) - assertEquals(BuildConfig.pythonFreeThreaded, props["freeThreaded"].toBoolean()) + assertEquals("true", props["freeThreaded"], "the desktop jar embeds the free-threaded CPython (#158)") } } diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt new file mode 100644 index 000000000..9d9eb1967 --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt @@ -0,0 +1,186 @@ +package python.multiplatform.ffi + +import java.io.File +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import kotlin.system.exitProcess +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * SPEC L-1a (issue #339), end to end: what a consumer app sees on its process stdout when it ends + * the interpreter with the public `Python3.finalize()` or flushes it with `Python3.flush()`. + * + * Each case is a child JVM. Finalizing destroys the interpreter every other test in this JVM shares, + * and the point is the process's real stdout (a file here, so Python's `sys.stdout` is block + * buffered exactly as in a packaged app whose output is redirected). The child is launched the way + * `PackagedPythonHomeLaunchTest`'s environment-`PYTHONHOME` guard launches its probe. + * + * Red before #339: [ShutdownProbe] calls `Python3.flush()`, which did not exist, so this file did not + * compile (a compile error, not a behavioural failure). [withoutFlushTheBufferedLineIsLost] is the + * control that shows the test can fail: the same probe without the flush loses the line. + */ +class InterpreterShutdownTest { + + private val scratchDirs = mutableListOf() + + @AfterTest + fun cleanup() { + scratchDirs.forEach { dir -> + if (Files.exists(dir.toPath(), java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + Files.walk(dir.toPath()).use { paths -> + paths.sorted(Comparator.reverseOrder()).forEach { Files.delete(it) } + } + } + } + scratchDirs.clear() + } + + /** Under `build/`, not the system temporary directory (AGENTS.md rule 2). */ + private fun newScratchDir(): File = + File("build/tmp/interpreterShutdownTest/${java.util.UUID.randomUUID()}").absoluteFile + .also { it.mkdirs(); scratchDirs += it } + + private class Outcome(val exitCode: Int, val output: String) + + private fun launch(scenario: String): Outcome { + val home = assertNotNull(System.getenv("PYTHONHOME"), "PYTHONHOME is unset for desktopTest") + val workDir = newScratchDir() + val log = File(workDir, "probe.log") + val java = File(System.getProperty("java.home"), "bin/java").path + val command = listOf( + java, + "--enable-preview", + "--enable-native-access=ALL-UNNAMED", + "-Djava.library.path=.", + "-cp", + System.getProperty("java.class.path"), + ShutdownProbe::class.java.name, + scenario, + ) + val builder = ProcessBuilder(command) + .directory(workDir) + .redirectErrorStream(true) + .redirectOutput(log) + builder.environment().remove("PYTHON_MULTIPLATFORM_LIBPYTHON") + // Python would line-buffer or not buffer at all with this set, and the test would prove nothing. + builder.environment().remove("PYTHONUNBUFFERED") + builder.environment()["PYTHONHOME"] = home + val process = builder.start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + return Outcome(-1, "timed out after 120 s\n" + log.readText()) + } + return Outcome(process.exitValue(), log.readText()) + } + + @Test + fun finalizeFlushesBufferedOutputAndReportsStatus() { + val outcome = launch("finalize") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue(ShutdownProbe.BUFFERED_LINE in outcome.output, "Python's buffered stdout was lost:\n${outcome.output}") + assertTrue("FINALIZED status=0 initialized=false" in outcome.output, outcome.output) + } + + @Test + fun flushWithoutFinalizeKeepsBufferedOutput() { + val outcome = launch("flush") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue(ShutdownProbe.BUFFERED_LINE in outcome.output, "Python's buffered stdout was lost:\n${outcome.output}") + } + + @Test + fun withoutFlushTheBufferedLineIsLost() { + // Control: if this ever finds the line, stdout is not buffered in the child and the two + // tests above no longer prove anything. + val outcome = launch("noflush") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("NOFLUSH_DONE" in outcome.output, outcome.output) + assertFalse(ShutdownProbe.BUFFERED_LINE in outcome.output, "stdout was not buffered:\n${outcome.output}") + } + + @Test + fun finalizeFromAnotherThreadIsRefusedAndTheInterpreterSurvives() { + val outcome = launch("otherthread") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("OTHER_THREAD_REFUSED" in outcome.output, outcome.output) + assertTrue("STILL_RUNNING 42" in outcome.output, outcome.output) + assertTrue("FINALIZED status=0 initialized=false" in outcome.output, outcome.output) + } + + @Test + fun finalizeWhenNotRunningReturnsZero() { + val outcome = launch("notrunning") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("NOT_RUNNING status=0" in outcome.output, outcome.output) + } +} + +/** The child's `main`; `args[0]` names the scenario. Kotlin output always goes through `println` + flush. */ +object ShutdownProbe { + const val BUFFERED_LINE = "PMP_BUFFERED_LINE_339" + + private fun writeBuffered() { + // No flush=True and no newline-triggered flush: the stream is a file, so it is block buffered. + Python3.exec("import sys\nsys.stdout.write('$BUFFERED_LINE\\n')") + } + + private fun say(line: String) { + println(line) + System.out.flush() + } + + @JvmStatic + fun main(args: Array) { + when (args.single()) { + "finalize" -> { + Python3.initialize(silent = true) + writeBuffered() + val status = Python3.finalize(silent = true) + say("FINALIZED status=$status initialized=${Python3.isInitialized}") + exitProcess(0) + } + "flush" -> { + Python3.initialize(silent = true) + writeBuffered() + Python3.flush() + // halt: no shutdown hooks, nothing else gets a chance to flush on our behalf. + Runtime.getRuntime().halt(0) + } + "noflush" -> { + Python3.initialize(silent = true) + writeBuffered() + say("NOFLUSH_DONE") + Runtime.getRuntime().halt(0) + } + "otherthread" -> { + Python3.initialize(silent = true) + var refused = false + val thread = Thread { + try { + Python3.finalize(silent = true) + } catch (e: IllegalStateException) { + refused = true + } + } + thread.start() + thread.join() + if (refused) say("OTHER_THREAD_REFUSED") + say("STILL_RUNNING ${Python3.eval("6 * 7")}") + val status = Python3.finalize(silent = true) + say("FINALIZED status=$status initialized=${Python3.isInitialized}") + exitProcess(0) + } + "notrunning" -> { + val status = Python3.finalize(silent = true) + say("NOT_RUNNING status=$status") + exitProcess(0) + } + else -> exitProcess(2) + } + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt index d0dd7e3c0..16ff5453f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt @@ -18,8 +18,8 @@ import kotlin.test.assertTrue * Before the copy-on-write cache the memo was a plain `mutableMapOf`. Two threads putting at once * could lose an entry (a later lookup then builds a second [PyType] for the same pointer, so * identity differs across threads) or corrupt the map outright (an exception). That needs real - * parallel writers, so it can fail only on a free-threaded CPython (`-PpythonFreeThreaded=true`); - * under the default GIL, getInstance's own `withPython` calls serialise most of the interleaving + * parallel writers, so it can fail only on a free-threaded CPython (desktop's build since issue + * #158); under a GIL build, getInstance's own `withPython` calls serialise most of the interleaving * and the test is a regression guard rather than a reproducer. * * Asserted: no thread throws; for each type every thread was handed the same instance; a final diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 99493fe05..c3f33358f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -15,7 +15,7 @@ import kotlin.test.BeforeTest import python.native.ffi.bindings import python.native.ffi.Panama import python.native.ffi.toNativePointer -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython class Node(var ref: PyObject? = null) { var rawPtr: Long = 0L @@ -146,7 +146,7 @@ class CycleCollectionTest { * with the global lock, and why the free-threaded branch is not just an offset fix. */ private fun typeRefCount(addr: Long): Long = - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { getLongMethod.invoke(theUnsafe, addr) as Long } else { val local = (getIntMethod.invoke(theUnsafe, addr + 12) as Int).toLong() and 0xFFFF_FFFFL @@ -383,7 +383,7 @@ class CycleCollectionTest { // `Py_REFCNT`, for whichever object header this build actually has. See the KDoc above: // offset 0 is `ob_refcnt` only on a build with the global lock. fun typeRefCount(): Long = - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { getLong.invoke(theUnsafe, proxyTypeAddr) as Long } else { val local = (getInt.invoke(theUnsafe, proxyTypeAddr + 12) as Int).toLong() and 0xFFFF_FFFFL @@ -395,7 +395,7 @@ class CycleCollectionTest { // carries, so that `before` and `alive` are read in the same state and the difference // between them is the instantiation loop and nothing else. On a build with the global // lock the count is always accurate and this would only cost a heap walk. - if (BuildConfig.pythonFreeThreaded) python.native.ffi.PyGC_Collect() + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount val typeRefBefore = typeRefCount() @@ -421,7 +421,7 @@ class CycleCollectionTest { ) // The instantiations above were deferred on a free-threaded build; this is what turns // them into countable references. Without it `alive` reads exactly `before`. - if (BuildConfig.pythonFreeThreaded) python.native.ffi.PyGC_Collect() + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val typeRefAlive = typeRefCount() assertEquals( diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt index d19b2dae0..691c43e70 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.PythonTestFixture @@ -34,6 +34,18 @@ import kotlin.test.assertTrue * Both tests below are matched pairs: each manipulates something that the free-threaded source * says is load-bearing and the global-lock source says is irrelevant, and asserts exactly that * split. If CPython changes either gate, they fail and say which one. + * + * ### CPython 3.15 removed the process-memory gate + * + * Desktop embeds 3.15.0t since issue #158, and [freeThreadedSchedulingTracksWholeProcessFootprintNotThePythonHeap] + * did what it was written to do: it failed when the gate changed. At the v3.15.0 tag + * `gc_should_collect_mem_usage`, `get_process_mem_usage`, `last_mem` and `deferred_count` are gone + * (`Python/gc_free_threading.c`, `Include/internal/pycore_interp_structs.h`), and + * `gc_should_collect` (`:1996`) returns true once `young.count` clears the threshold and + * `long_lived_total / 4` (`:2009`). The `old[0].threshold == 0` short circuit is still there + * (`:2004`), and `gc_collect_main` still re-asks `gc_should_collect` for a heap-triggered + * collection (`:2217`). The table above is the 3.14.7 record; the first test's assertions hold on + * both, the second asserts the 3.15 shape. */ class FreeThreadedGCGateTest { @@ -57,10 +69,10 @@ class FreeThreadedGCGateTest { /** * `gc.collect()`. * - * Free-threaded this also **resets `last_mem` to the current process footprint** - * (`gc_free_threading.c:2301`, reached because `_Py_GC_REASON_MANUAL` skips the gate at - * `:2328`), which is what makes the arms below independent of each other: every arm starts - * from a freshly pinned memory baseline. + * It resets `young.count` and sets `long_lived_total` to what survived, which is what makes + * the arms below independent of each other: every arm starts from the same scheduling state. + * On 3.14t it also reset `last_mem` to the current process footprint (`gc_free_threading.c:2301` + * at 3.14.7), the memory baseline the second test's 3.14 form depended on. */ private fun forceCollect(gc: PyObject) { val collect = gc.getAttr("collect") @@ -69,7 +81,8 @@ class FreeThreadedGCGateTest { } /** `gc.get_stats()[0]['collections']` -- incremented only *past* the gate at - * `gc_free_threading.c:2328`, so it counts collections that actually ran, not bits set. */ + * `gc_free_threading.c:2217` (v3.15.0; `:2328` at 3.14.7), so it counts collections that + * actually ran, not bits set. */ private fun gen0Collections(gc: PyObject): Long { val f = gc.getAttr("get_stats") val t = f.invoke() @@ -113,7 +126,7 @@ class FreeThreadedGCGateTest { buildAndDropCycles(builtins, cyclesCreated / 2) val residue = countTrackedObjects(gc) - initial println( - "GATE[$label] freeThreaded=${BuildConfig.pythonFreeThreaded} residue=$residue/$totalObjects " + + "GATE[$label] freeThreaded=${embedsFreeThreadedPython} residue=$residue/$totalObjects " + "collectionsThatActuallyRan=${gen0Collections(gc) - collectionsBefore}" ) forceCollect(gc) @@ -124,7 +137,7 @@ class FreeThreadedGCGateTest { * `gc.set_threshold(_, 0, _)` -- a setting about **generation 1** -- is what decides whether * **generation 0** is collectable at all on the free-threaded build. * - * `gc_free_threading.c:2126` short-circuits `gc_should_collect` to `true` when + * `gc_free_threading.c:2004` (v3.15.0; `:2126` at 3.14.7) short-circuits `gc_should_collect` to `true` when * `old[0].threshold == 0`, skipping both `long_lived_total/4` and the process-memory gate. * Nothing in `gc.c` reads that field while scheduling, so the same change must be a no-op with * the global lock. Run A/B/A in one JVM so the §8b latch cannot account for the difference. @@ -152,17 +165,17 @@ class FreeThreadedGCGateTest { setThreshold(2000, 10, 10) val defaultAfter = arm(gc, builtins, "default-thresholds-again") - // Holds on both builds, for different reasons: free-threaded because :2126 + // Holds on both builds, for different reasons: free-threaded because :2004 // skips the gates, GIL-side because there was never a gate to skip. assertTrue( bypassed < reclaimedBound, "with gc.set_threshold(2000, 0, 0) the eval-loop checkpoint should reclaim " + "the bulk of $totalObjects dropped objects, but $bypassed remained. " + - "Free-threaded that means gc_free_threading.c:2126 no longer " + + "Free-threaded that means gc_free_threading.c:2004 (v3.15.0) no longer " + "short-circuits gc_should_collect on old[0].threshold == 0" ) - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { // gc.c:1866 schedules on generations[0].count alone and gc.c:1258 selects // generation 0 on count alone, so generation 1's threshold cannot matter. assertTrue( @@ -183,27 +196,33 @@ class FreeThreadedGCGateTest { } /** - * Free-threaded, whether the Python cyclic collector runs is a function of the **JVM's** - * memory, not of Python's. + * Whether the Python cyclic collector runs must not depend on the **JVM's** memory. * - * `gc_should_collect_mem_usage` (`gc_free_threading.c:2080`) asks whether the *whole process* - * footprint has grown by more than `last_mem / 10` since the last collection. So faulting in - * 400 MB of ballast partway through the workload -- changing no threshold, allocating no extra - * Python container, touching nothing the collector tracks -- should open the gate, and - * dropping the ballast should shut it again. `gc.c` has no such gate, so the identical - * manipulation must be a no-op with the global lock. + * Faulting in 400 MB of ballast partway through the workload changes no threshold, allocates + * no extra Python container and touches nothing the collector tracks. On 3.14t that opened + * `gc_should_collect_mem_usage` (`gc_free_threading.c:2080` at 3.14.7), which asked whether + * the *whole process* footprint had grown by more than `last_mem / 10` since the last + * collection. Measured on 3.14t (3 consecutive JVMs, identical to the object), residue out of + * 20,000: * - * Measured (3 consecutive JVMs, identical to the object), residue out of 20,000: - * - * | arm | free-threaded | global lock | + * | arm | free-threaded 3.14t | global lock | * |---|---:|---:| * | baseline | 20,000 (0 collections) | 1,972 (9 collections) | * | 400 MB faulted in midway | **9,754 (1 collection)** | 1,988 (9 collections) | * | ballast dropped | 20,000 (0 collections) | 1,968 (9 collections) | * - * The 400 MB figure assumes the test JVM's footprint is under ~4 GB, so that 400 MB clears the - * `last_mem / 10` bar; the failure message says so, because a much larger heap would make this - * test wrong rather than the mechanism wrong. + * 3.15.0 deleted that gate (see the class KDoc), so on the 3.15t desktop embeds the ballast + * must be a no-op, as it always was with the global lock. Each arm starts from a forced + * collection and runs the identical workload, so the scheduling inputs that remain + * (`young.count`, the threshold, `long_lived_total`) are the same in all three and so is the + * outcome. What is asserted free-threaded is exactly that: the ballast arm and the + * ballast-dropped arm land within a quarter of the workload of the baseline. The bound is not + * "everything was reclaimed", because whether `long_lived_total / 4` lets this process collect + * at all depends on how many objects the test JVM's interpreter already holds, which this test + * does not control. + * + * What it still catches: a process-memory gate coming back. The 3.14t row above differs from + * its own baseline by 10,246 objects, more than twice the 5,000 allowed. */ @Test fun freeThreadedSchedulingTracksWholeProcessFootprintNotThePythonHeap() = @@ -220,19 +239,20 @@ class FreeThreadedGCGateTest { Python3.exec("del __pm_ballast") val dropped = arm(gc, builtins, "ballast-dropped") - if (BuildConfig.pythonFreeThreaded) { - // The whole finding: JVM memory growth, and nothing else, collected Python - // cycles. Deliberately not a tight bound -- how much a single collection - // takes depends on where in the workload the footprint crossed the bar. + if (embedsFreeThreadedPython) { + // 3.15: no process-memory input, so the ballast must not change the + // outcome. A quarter of the workload absorbs a collection landing a few + // objects earlier or later in one arm; it does not absorb 3.14t's 10,246. + val tolerance = totalObjects / 4 assertTrue( - withBallast < baseline, - "faulting in 400 MB of process memory partway through the workload " + - "should have opened gc_should_collect_mem_usage " + - "(gc_free_threading.c:2100) and let the checkpoint collect, but " + - "the ballast arm left $withBallast against the baseline's " + - "$baseline out of $totalObjects. Either that gate is gone, or " + - "this JVM's footprint is large enough (>~4 GB) that 400 MB no " + - "longer clears last_mem/10" + kotlin.math.abs(withBallast - baseline) < tolerance && + kotlin.math.abs(dropped - baseline) < tolerance, + "free-threaded 3.15.0 has no process-memory gate (gc_should_collect, " + + "gc_free_threading.c:1996), so faulting in and dropping 400 MB of " + + "process memory should not change what the checkpoint collects, but " + + "the arms left $baseline / $withBallast / $dropped out of " + + "$totalObjects (allowed spread: $tolerance). A gate on whole-process " + + "memory, like 3.14's gc_should_collect_mem_usage, is back" ) } else { // gc.c never calls get_process_mem_usage and has no last_mem field at all. diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt index d08904416..fb3863b9a 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.PythonTestFixture import kotlin.test.Test @@ -11,9 +11,11 @@ import kotlin.test.assertTrue * build specifically: `_PY_GC_SCHEDULED_BIT` is read only by the eval-loop checkpoint, on both * builds, so an embedder that never runs Python bytecode never runs the cyclic collector either * -- with or without free threading. `autoDrainInterval` is saved and restored around each test; - * the default (0 on this build) is never changed by running this file. + * the default (0 on a GIL build, 32 on the free-threaded one) is never changed by running this file. * - * Also runs, unmodified, on `-PpythonFreeThreaded=true`, where the automatic checkpoint reliably + * Desktop embeds the free-threaded build since issue #158, so the GIL branches below no longer run + * on desktop; they are kept for the record of what the GIL build measured. On the free-threaded + * build the automatic checkpoint reliably * fires but does not reliably reclaim: under this heavy single-thread, C-API-only workload the * residue swings between the GIL build's ~1,970 and the full ~20,000, run to run, so only the GIL * build gets a bound here. That swing is not a mystery any more -- it tracks the *JVM's* memory @@ -165,7 +167,7 @@ class GCSchedulingMeasurementTest { * The residue bound is pinned on the GIL build only, where it is stable at ~1,920-1,970 out of * ~20,000 in every configuration measured. * - * Free-threaded (`-PpythonFreeThreaded=true`, where `autoDrainInterval = 32` is already the + * Free-threaded (desktop's only build since issue #158, where `autoDrainInterval = 32` is already the * shipped default rather than something this test turns on) it is not stable, and no bound is * asserted. `CheckpointCounter.reached` climbs by an identical, deterministic amount on every * run, but the residue does not: measured 20,000 -- i.e. nothing reclaimed at all -- in 10 of @@ -222,7 +224,7 @@ class GCSchedulingMeasurementTest { val delta = after - initial println( - "--- MEASUREMENT 2: Interval 32 (freeThreaded=${BuildConfig.pythonFreeThreaded}) --- " + + "--- MEASUREMENT 2: Interval 32 (freeThreaded=${embedsFreeThreadedPython}) --- " + "initial=$initial after=$after delta=$delta checkpointsReached=$reachedDelta" ) @@ -232,7 +234,7 @@ class GCSchedulingMeasurementTest { "$cyclesCreated outermost withGIL scopes, but none were reached" ) - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { // Each outermost withGIL scope inside buildAndDropCycles counts down; at interval // 32 that is thousands of checkpoints over 10,000 rounds, each one able to run the // scheduled collection. The residue should be bounded by roughly one interval's diff --git a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt index 0a1cab1a4..cf05640d6 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt @@ -28,8 +28,8 @@ class SidecarLibraryLookupTest { "PYTHONHOME is unset for desktopTest; the sidecar lookup has nothing to resolve against" ) - // `taggedVersionString`, not `compactVersionString`: under `-PpythonFreeThreaded=true` - // PYTHONHOME contains `libpython3.14t.dylib` and no `libpython3.14.dylib`, so asking for + // `taggedVersionString`, not `compactVersionString`: desktop embeds the free-threaded + // build (issue #158), so PYTHONHOME contains `libpython3.15t.dylib` and no `libpython3.15.dylib`, so asking for // the un-suffixed name resolves nothing. This mirrors what `manager.loadLibPython` builds. val tagged = Versions.currentVersion.taggedVersionString val libraryName = if (currentPlatform.os == OSType.Windows) { diff --git a/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt new file mode 100644 index 000000000..d39724875 --- /dev/null +++ b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** iOS embeds the project's own free-threaded Python.xcframework, 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt new file mode 100644 index 000000000..4ec57ebf7 --- /dev/null +++ b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt @@ -0,0 +1,33 @@ +package python.multiplatform + +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-2a (issue #328): iOS links the project's own free-threaded `Python.xcframework` (3.15t), + * the same archive the plugin's iOS consumer wiring stages. + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `iosMain`, so `abiFlags` was `""` + * and `IosPythonHome`/`PythonHomeCheck` looked for `lib/python3.15`, and the library's cinterop and + * test binary used python.org's GIL framework (`Py_GIL_DISABLED` 0). (This replaces + * `IosGilBuildTest`, which pinned the GIL build until #328.) + */ +class IosFreeThreadedBuildTest { + + @Test + fun iosReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "iOS embeds CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLinkedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the iOS framework is a GIL build; iOS must embed the free-threaded one (#328)", + ) + } +} diff --git a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt index 81b5fa356..258336660 100644 --- a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt +++ b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt @@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap * * `desktopMain` implements these over cached, unbound Panama `MethodHandle`s (one per * shape, see `Panama.kt`). `androidMain` implements these by delegating through - * JNI to Kotlin/Native trampolines compiled into `libmultiplatform_python3.13.so` (see + * JNI to Kotlin/Native trampolines compiled into `libmultiplatform_python3.15t.so` (see * `nativeMain/.../EmbedAPI.native.kt`), which invoke the target function pointer directly * via a `CFunction` cast -- no NDK, no C project, no ART entry-point patching. * diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt new file mode 100644 index 000000000..dfa1251af --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt @@ -0,0 +1,39 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import platform.posix.uname +import platform.posix.utsname + +/** + * The Linux Kotlin/Native (`linuxX64`) view of the running host. + */ +object LinuxNativePlatform : Platform { + override val os = OSType.Linux + override val arch: String = machineName() + override val version: String = kernelRelease() + override val versionCode: Int? = null + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun machineName(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.machine.toKString().ifEmpty { "unknown" } +} + +@OptIn(ExperimentalForeignApi::class) +private fun kernelRelease(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.release.toKString().ifEmpty { "unknown" } +} + +actual val currentPlatform: Platform = LinuxNativePlatform diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt new file mode 100644 index 000000000..94cbe50da --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt @@ -0,0 +1,9 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.convert + +/** glibc's `pthread_t` is a plain unsigned integral typedef (`unsigned long`), not a pointer. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self().convert() diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt new file mode 100644 index 000000000..72e10a9ea --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt @@ -0,0 +1,21 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ByteVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.allocArray +import kotlinx.cinterop.convert +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.set +import kotlinx.cinterop.toKString +import platform.posix.PATH_MAX +import platform.posix.readlink + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? = memScoped { + val buf = allocArray(PATH_MAX) + val len = readlink("/proc/self/exe", buf, (PATH_MAX - 1).convert()) + if (len <= 0) return null + buf[len.toInt()] = 0 + val path = buf.toKString() + path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt new file mode 100644 index 000000000..6be1e9ab1 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt @@ -0,0 +1,49 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import kotlinx.cinterop.useContents +import platform.Foundation.NSProcessInfo +import platform.posix.uname +import platform.posix.utsname + +/** + * The macOS Kotlin/Native (`macosArm64`) view of the running host. + */ +object MacosNativePlatform : Platform { + override val os = OSType.MacOS + override val arch: String = getDeviceArch() + override val version: String = getOSVersionString() + override val versionCode: Int = getOSVersionCode() + override val versionText = "SDK $versionCode" + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun getDeviceArch(): String = memScoped { + val systemInfo = alloc() + if (uname(systemInfo.ptr) != 0) { + throw RuntimeException("Failed to get device arch info") + } + systemInfo.machine.toKString() +} + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionString(): String = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + "$majorVersion.$minorVersion.$patchVersion" + } + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionCode(): Int = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + majorVersion.toInt() * 10_000 + minorVersion.toInt() * 100 + patchVersion.toInt() + } + +actual val currentPlatform: Platform = MacosNativePlatform diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt new file mode 100644 index 000000000..d19062749 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt @@ -0,0 +1,8 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi + +/** Darwin's `pthread_self()` returns a pointer; its bit pattern is the thread identity. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self()?.rawValue?.toLong() ?: 0L diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt new file mode 100644 index 000000000..38fb7176b --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt @@ -0,0 +1,27 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.toKString +import platform.Foundation.NSBundle +import platform.Foundation.NSProcessInfo +import platform.posix.realpath + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? { + val raw = NSBundle.mainBundle.executablePath + ?: (NSProcessInfo.processInfo.arguments.firstOrNull() as? String) + ?: return null + val resolved = realpath(raw, null) + val path = if (resolved != null) { + try { + resolved.toKString() + } finally { + nativeHeap.free(resolved) + } + } else { + raw + } + return path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 06c590c5b..ebe3f78bd 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -4,6 +4,7 @@ package python.multiplatform.ref import kotlinx.cinterop.* import python.multiplatform.currentThreadId +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.ProxyTypeFactory import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.PythonTestFixture @@ -173,21 +174,38 @@ class CycleCollectionTest { * proxy, and the number of proxies that were still owed if one stops doing so. */ private fun settleKotlinFinalisation(typeAddr: Long): Long { - val header = typeAddr.toCPointer() ?: return 0L - val started = header.pointed.value + if (typeAddr == 0L) return 0L + val started = typeRefCount(typeAddr) var last = started var quiet = 0 var turns = 0 while (quiet < 3 && turns < 50) { kotlin.native.runtime.GC.collect() platform.posix.usleep(2_000u) - val now = header.pointed.value + val now = typeRefCount(typeAddr) if (now == last) quiet++ else { quiet = 0; last = now } turns++ } return started - last } + /** + * `Py_REFCNT` of the object at [addr], for whichever object header this build has. On a GIL + * build `ob_refcnt` is the first field. On the free-threaded 3.15t, which every Kotlin/Native + * target here embeds (desktop #158, iOS and androidNative #328), offset 0 is `ob_tid` and the + * count is `ob_ref_local` (uint32 at +12) plus `ob_ref_shared` (at +16) shifted right by + * `_Py_REF_SHARED_SHIFT` (2). Same probe as the desktop `CycleCollectionTest.typeRefCount`. + * A plain, racy load, which is all the callers need. + */ + private fun typeRefCount(addr: Long): Long = + if (!embedsFreeThreadedPython) { + assertNotNull(addr.toCPointer()).pointed.value + } else { + val local = assertNotNull((addr + 12).toCPointer()).pointed.value.toLong() + val shared = assertNotNull((addr + 16).toCPointer()).pointed.value + local + (shared shr 2) + } + @Test fun testCycleCollectionByGC() = PythonTestFixture.withInterpreter { withGIL { @@ -320,8 +338,15 @@ class CycleCollectionTest { * releases it. Kotlin/Native's `proxyDealloc` hand-writes that (`PyObject_Type` for the * temporary, then two `Py_DecRef`s), so it can leak one type reference per instance by * forgetting, or free the type out from under the process by doing it twice. Both are checked - * by reading `ob_refcnt`, the first field of `PyObject` and fixed there by the stable ABI -- - * `sys.getrefcount` cannot be used because it counts its own argument's temporary. + * by reading the type's reference count out of its object header -- `sys.getrefcount` cannot + * be used because it counts its own argument's temporary. + * + * The header depends on the build. On a GIL build `ob_refcnt` is the first field. Every + * Kotlin/Native target here embeds the free-threaded 3.15t (desktop #158, iOS and androidNative + * #328), where offset 0 is `ob_tid` and the count is `ob_ref_local` (uint32 at +12) plus + * `ob_ref_shared` (at +16) shifted right by 2, and where a heap type's references are deferred + * until a collection materialises them, so one is taken on either side of the instantiation + * loop. Same probe and same reasoning as the desktop `CycleCollectionTest`. * * The "rises while alive" assertion is not decoration: it is what proves the probe reads a * real refcount, so the "returns afterwards" assertion cannot pass vacuously on an address @@ -334,12 +359,12 @@ class CycleCollectionTest { assertTrue(proxyTypeAddr != 0L, "Proxy type creation failed") val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - // The type object's own ob_refcnt, as a plain 64-bit load at the type's address. - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) - fun typeRefCount(): Long = typeHeader.pointed.value + // Free-threaded: materialise the deferred references the type already carries, so that + // `before` and `alive` are read in the same state. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount - val typeRefBefore = typeRefCount() + val typeRefBefore = typeRefCount(proxyTypeAddr) val rounds = 100 val proxies = LongArray(rounds) @@ -364,7 +389,10 @@ class CycleCollectionTest { liveBefore + rounds, HandleTable.liveCount, "setup failed: $rounds handles should be rooted while the proxies are alive", ) - val typeRefAlive = typeRefCount() + // The instantiations above were deferred on a free-threaded build; a collection is + // what turns them into countable references. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() + val typeRefAlive = typeRefCount(proxyTypeAddr) assertEquals( typeRefBefore + rounds, typeRefAlive, "each live instance of a heap type holds one reference to that type, so the count " + @@ -390,7 +418,7 @@ class CycleCollectionTest { "the table should be back to its starting size once every proxy is gone", ) assertEquals( - typeRefBefore, typeRefCount(), + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc must release the instance's reference to its heap type exactly once: " + "a count above $typeRefBefore means it was never released and the type leaks " + "per instance, below means it was released twice and the type will be freed " + @@ -440,10 +468,18 @@ class CycleCollectionTest { withGIL { val proxyTypeAddr = ProxyTypeFactory.createProxyType() val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) + + // Free-threaded: an instance takes its type reference through this thread's per-thread + // refcount array (`_Py_INCREF_TYPE`), while proxyDealloc gives it back with a header + // `Py_DecRef`, so the type is cached across tests and earlier instantiations on this + // thread can still be unmerged here. The worker's gc.collect() below merges every + // thread's array (`gc_free_threading.c:2090` at v3.15.0); collecting first puts the + // opening reading in that same merged state, so the comparison is instantiation and + // dealloc and nothing left over. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount - val typeRefBefore = typeHeader.pointed.value + val typeRefBefore = typeRefCount(proxyTypeAddr) val pyObjPtr = assertNotNull( PyObject_CallObject(proxyTypeAddr.toCPointer(), null), @@ -514,7 +550,7 @@ class CycleCollectionTest { ) assertEquals(liveBefore, HandleTable.liveCount, "the table did not return to its starting size") assertEquals( - typeRefBefore, typeHeader.pointed.value, + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc ran on a CPython-created thread but did not balance the instance's " + "reference to its heap type", ) diff --git a/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt b/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt new file mode 100644 index 000000000..798ce6f03 --- /dev/null +++ b/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** wasmJs runs a GIL CPython built for wasm32-emscripten until #175. See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = false diff --git a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt new file mode 100644 index 000000000..e634832d1 --- /dev/null +++ b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt @@ -0,0 +1,22 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Issue #158 moved desktop to the free-threaded CPython 3.15t, and #328 moved Android, androidNative + * and iOS; wasmJs still embeds the GIL build until #175. This pins that neither change leaked here. + * + * Not red before the change: wasmJs was GIL before and is GIL after. It is the guard against the + * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which + * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that wasmJs does not ship. + */ +class WasmGilBuildTest { + + @Test + fun wasmStillReportsTheGilAbi() { + val v = Versions.currentVersion + assertEquals("", v.abiFlags, "wasmJs embeds the GIL CPython until #175") + assertEquals(v.compactVersionString, v.taggedVersionString) + } +} diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 2e8a35dcd..0b8d39e0d 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -350,8 +350,8 @@ val hostPlatform = when { // the same way rather than hardcoding a flat path -- otherwise the sample points PYTHONHOME at a // directory that only exists for the default configuration. val samplePythonVersion = project.findProperty("pythonVersion")?.toString() ?: rootProject.version.toString() -val sampleFlavourSuffix = - if (project.findProperty("pythonFreeThreaded")?.toString()?.toBoolean() == true) "-freethreaded" else "" +// Desktop acquires only the free-threaded build (#158), unpacked under `-freethreaded`. +val sampleFlavourSuffix = "-freethreaded" val pythonHomeForHost = project(":python-multiplatform").layout.buildDirectory .dir("python-standalone/extracted/$samplePythonVersion/$hostPlatform$sampleFlavourSuffix/python") @@ -407,14 +407,19 @@ tasks.matching { it.name == "run" }.configureEach { val nativeImageJavaLauncher = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) } val nativeImageOutputDir = layout.buildDirectory.dir("native/nativeCompile") +pythonBindings { + nativeImageDirectory.set(nativeImageOutputDir) +} + val nativeCompile by tasks.registering(Exec::class) { group = "native" description = "Builds upcall-native-demo with GraalVM native-image from NativeImageMain.kt" - dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar) + dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar, "stageNativeImagePythonHome") val classpathProvider = files( desktopMainCompilation.output.allOutputs, @@ -425,7 +430,7 @@ val nativeCompile by tasks.registering(Exec::class) { val launcher = nativeImageJavaLauncher inputs.files(classpathProvider) - outputs.dir(outDir) + outputs.file(outDir.map { it.file("upcall-native-demo") }) doFirst { outDir.get().asFile.mkdirs() @@ -445,8 +450,10 @@ tasks.register("runNativeUpcallDemo") { group = "verification" description = "Runs the native-image binary built by nativeCompile and checks the upcall path" dependsOn(nativeCompile) + workingDir(nativeImageOutputDir) executable = nativeImageOutputDir.get().asFile.resolve("upcall-native-demo").absolutePath - environment("PYTHONHOME", pythonHomeForHost.get().asFile.absolutePath) + environment.remove("JAVA_HOME") + environment.remove("PYTHONHOME") // A zero exit status is not the same as a working upcall path. `PyRun_SimpleString` reports // failure for an exception raised inside the script, but a Python interpreter that never got