From fb5b531c50bff13c2f672e95604cba253d72abf8 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 4 Oct 2026 00:10:52 +0900 Subject: [PATCH 01/61] Bench: Add Cython (pure and typed) and numba columns to the TypedPython benchmarks --- .gitignore | 7 + .../typedpython/cython/binary_trees.pyx | 46 ++ benchmarks/typedpython/cython/build.py | 46 ++ benchmarks/typedpython/cython/fannkuch.pyx | 61 +++ benchmarks/typedpython/cython/nbody.pyx | 80 ++++ .../typedpython/cython/spectral_norm.pyx | 61 +++ benchmarks/typedpython/cython/wordfreq.pyx | 54 +++ benchmarks/typedpython/numba/binary_trees.py | 72 +++ benchmarks/typedpython/numba/fannkuch.py | 62 +++ benchmarks/typedpython/numba/nbody.py | 82 ++++ benchmarks/typedpython/numba/spectral_norm.py | 61 +++ benchmarks/typedpython/numba/wordfreq.py | 69 +++ benchmarks/typedpython/run.py | 436 +++++++++++++++--- 13 files changed, 1083 insertions(+), 54 deletions(-) create mode 100644 benchmarks/typedpython/cython/binary_trees.pyx create mode 100644 benchmarks/typedpython/cython/build.py create mode 100644 benchmarks/typedpython/cython/fannkuch.pyx create mode 100644 benchmarks/typedpython/cython/nbody.pyx create mode 100644 benchmarks/typedpython/cython/spectral_norm.pyx create mode 100644 benchmarks/typedpython/cython/wordfreq.pyx create mode 100644 benchmarks/typedpython/numba/binary_trees.py create mode 100644 benchmarks/typedpython/numba/fannkuch.py create mode 100644 benchmarks/typedpython/numba/nbody.py create mode 100644 benchmarks/typedpython/numba/spectral_norm.py create mode 100644 benchmarks/typedpython/numba/wordfreq.py diff --git a/.gitignore b/.gitignore index 670457d97..c9ffafa6b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,10 @@ google-services.json # scratch space for agents and scripts; delete when done .tmp/ + +# TypedPython benchmark harness: uv venv, Cython/numba build output and numba caches +benchmarks/typedpython/.venv/ +benchmarks/typedpython/build/ +benchmarks/typedpython/target/ +benchmarks/typedpython/java-out/ +benchmarks/typedpython/tp-build/ diff --git a/benchmarks/typedpython/cython/binary_trees.pyx b/benchmarks/typedpython/cython/binary_trees.pyx new file mode 100644 index 000000000..1f25e4e66 --- /dev/null +++ b/benchmarks/typedpython/cython/binary_trees.pyx @@ -0,0 +1,46 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/binary_trees.py: one heap object per node, reference counted, freed by +# refcounting (not an arena). Semantics that differ from CPython: Node is a cdef class (no __dict__, +# attributes not visible from Python), C long counters, recursion uses the C stack (no +# RecursionError), no None checks on typed attributes. +import sys + + +cdef class Node: + cdef Node left + cdef Node right + + def __init__(self, Node left, Node right): + self.left = left + self.right = right + + +cdef Node make(int depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +cdef long check(Node node) noexcept: + cdef Node left = node.left + cdef Node right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +def main(): + cdef int n = int(sys.argv[1]) + cdef int min_depth = 4 + cdef int max_depth = max(min_depth + 2, n) + cdef int depth, i + cdef long iterations, chk, total = 0 + cdef long stretch = check(make(max_depth + 1)) + cdef Node long_lived = make(max_depth) + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for i in range(iterations): + chk += check(make(depth)) + total += chk + print(f"{stretch} {total} {check(long_lived)}") diff --git a/benchmarks/typedpython/cython/build.py b/benchmarks/typedpython/cython/build.py new file mode 100644 index 000000000..1ed61b65a --- /dev/null +++ b/benchmarks/typedpython/cython/build.py @@ -0,0 +1,46 @@ +"""Build the Cython columns. Run with the harness venv's python (see ../README in run.py's docstring). + + .venv/bin/python cython/build.py + +Writes /pure/.so (the unmodified py/*.py compiled in pure Python mode, annotations +as written) and /typed/_typed.so (cython/*.pyx). Not part of any timed run. +Flags: -O3 -ffp-contract=off (no fused multiply-add, so float results match CPython bit for bit). +""" +import shutil +import sys +from pathlib import Path + +from Cython.Build import cythonize +from setuptools import Extension, setup + +HERE = Path(__file__).resolve().parent +PY = HERE.parent / "py" +NAMES = {"nbody": "nbody", "spectral_norm": "spectral_norm", "fannkuch": "fannkuch", + "binary_trees": "binary_trees", "wordfreq": "wordfreq"} +CFLAGS = ["-O3", "-ffp-contract=off"] + + +def build(out: Path, kind: str) -> None: + work = out / f"src-{kind}" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + exts = [] + for name in NAMES: + if kind == "pure": + src = work / f"{name}.py" + shutil.copy(PY / f"{name}.py", src) # unmodified copy + mod = name + else: + mod = f"{name}_typed" + src = work / f"{mod}.pyx" + shutil.copy(HERE / f"{name}.pyx", src) + exts.append(Extension(mod, [str(src)], extra_compile_args=CFLAGS)) + ext_modules = cythonize(exts, language_level=3, quiet=True, build_dir=str(work / "c")) + setup(name=f"tp-bench-{kind}", ext_modules=ext_modules, script_name="setup.py", + script_args=["build_ext", "--build-lib", str(out / kind), "--build-temp", str(work / "obj"), "-q"]) + + +if __name__ == "__main__": + target = Path(sys.argv[1]).resolve() + for k in ("pure", "typed"): + build(target, k) diff --git a/benchmarks/typedpython/cython/fannkuch.pyx b/benchmarks/typedpython/cython/fannkuch.pyx new file mode 100644 index 000000000..9927c47fc --- /dev/null +++ b/benchmarks/typedpython/cython/fannkuch.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/fannkuch.py. Semantics that differ from CPython: C int arrays and C long +# counters (wrap instead of growing), no bounds checks, fixed capacity of 16 elements (n > 16 raises). +import sys + + +cdef (long, long) fannkuch(int n) noexcept: + cdef int perm1[16] + cdef int count[16] + cdef int perm[16] + cdef long max_flips = 0, checksum = 0, perm_count = 0, flips + cdef int r = n, i, k, lo, hi, t, p0 + for i in range(n): + perm1[i] = i + count[i] = 0 + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def main(): + cdef int n = int(sys.argv[1]) + if n > 16: + raise ValueError("fannkuch.pyx supports n <= 16") + cdef long checksum, max_flips + checksum, max_flips = fannkuch(n) + print(f"{checksum} {max_flips}") diff --git a/benchmarks/typedpython/cython/nbody.pyx b/benchmarks/typedpython/cython/nbody.pyx new file mode 100644 index 000000000..a12fd632a --- /dev/null +++ b/benchmarks/typedpython/cython/nbody.pyx @@ -0,0 +1,80 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/nbody.py. Semantics that differ from CPython: float division does not raise +# ZeroDivisionError (cdivision), no bounds checks (arrays are C arrays), loop counters are C long. +from libc.math cimport sqrt +import sys + +cdef double PI = 3.141592653589793 +cdef double SOLAR_MASS = 4 * PI * PI +cdef double DAYS_PER_YEAR = 365.24 + + +cdef void offset_momentum(double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef double px = 0.0, py = 0.0, pz = 0.0 + cdef int i + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +cdef void advance(double dt, long steps, double* x, double* y, double* z, + double* vx, double* vy, double* vz, double* mass) noexcept nogil: + cdef long s + cdef int i, j + cdef double dx, dy, dz, d2, mag + for s in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +cdef double energy(double* x, double* y, double* z, double* vx, double* vy, double* vz, + double* mass) noexcept nogil: + cdef double e = 0.0, dx, dy, dz + cdef int i, j + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def main(): + cdef long n = int(sys.argv[1]) + cdef double[5] x = [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01] + cdef double[5] y = [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01] + cdef double[5] z = [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01] + cdef double[5] vx = [0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR] + cdef double[5] vy = [0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR] + cdef double[5] vz = [0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR] + cdef double[5] mass = [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS] + offset_momentum(vx, vy, vz, mass) + cdef double before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + cdef double after = energy(x, y, z, vx, vy, vz, mass) + print(f"{before:.9f} {after:.9f}") diff --git a/benchmarks/typedpython/cython/spectral_norm.pyx b/benchmarks/typedpython/cython/spectral_norm.pyx new file mode 100644 index 000000000..0b73cd8bc --- /dev/null +++ b/benchmarks/typedpython/cython/spectral_norm.pyx @@ -0,0 +1,61 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/spectral_norm.py. Semantics that differ from CPython: loop and index +# arithmetic is C long (wraps instead of growing; not reachable at these sizes), cdivision (no +# ZeroDivisionError), no bounds checks (malloc'd C arrays). +from libc.math cimport sqrt +from libc.stdlib cimport malloc, free +import sys + + +cdef inline double eval_a(long i, long j) noexcept nogil: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +cdef void mul_av(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +cdef void mul_atv(long n, double* v, double* out) noexcept nogil: + cdef long i, j + cdef double s + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +cdef void mul_atav(long n, double* v, double* out, double* tmp) noexcept nogil: + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +def main(): + cdef long n = int(sys.argv[1]) + cdef long i + cdef double* u = malloc(n * sizeof(double)) + cdef double* v = malloc(n * sizeof(double)) + cdef double* tmp = malloc(n * sizeof(double)) + cdef double vbv = 0.0, vv = 0.0 + try: + for i in range(n): + u[i] = 1.0 + v[i] = 0.0 + tmp[i] = 0.0 + for i in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + finally: + free(u) + free(v) + free(tmp) + print(f"{sqrt(vbv / vv):.9f}") diff --git a/benchmarks/typedpython/cython/wordfreq.pyx b/benchmarks/typedpython/cython/wordfreq.pyx new file mode 100644 index 000000000..2b1734a91 --- /dev/null +++ b/benchmarks/typedpython/cython/wordfreq.pyx @@ -0,0 +1,54 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True, initializedcheck=False +# Same algorithm as py/wordfreq.py: the counts stay a Python dict[str, int] (the algorithm's data +# structure), only the arithmetic is typed and words are built in a C buffer. Semantics that differ +# from CPython: C long arithmetic (wraps; the LCG product stays below 2**47 and the weighted sum +# below 2**63 at benchmark size), cdivision. +import sys + +cdef long VOCAB = 50000 +cdef long MOD = 2147483647 + + +cdef inline long next_state(long s) noexcept nogil: + return (s * 48271) % MOD + + +cdef str word_of(long word_id): + cdef char buf[32] + cdef int pos = 32 + cdef long x = word_id + 17576 + while x > 0: + pos -= 1 + buf[pos] = (97 + x % 26) + x //= 26 + return (buf[pos:32]).decode("ascii") + + +def main(): + cdef long n = int(sys.argv[1]) + cdef dict counts = {} + cdef long s = 12345, a, b, i, c, weighted = 0, best_count + cdef str word, w, best_word + cdef list taken = [] + cdef list parts = [] + for i in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + counts[word] = counts.get(word, 0) + 1 + for w, c in counts.items(): + weighted += c * c + for i in range(5): + best_word = "" + best_count = -1 + for w, c in counts.items(): + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + parts.append(f"{best_word}:{best_count}") + print(f"{len(counts)} {weighted} {','.join(parts)}") diff --git a/benchmarks/typedpython/numba/binary_trees.py b/benchmarks/typedpython/numba/binary_trees.py new file mode 100644 index 000000000..35a4b7fcc --- /dev/null +++ b/benchmarks/typedpython/numba/binary_trees.py @@ -0,0 +1,72 @@ +"""numba version of py/binary_trees.py, using numba.experimental.jitclass. + +Node is a jitclass with a deferred (self-referential) Optional type, the closest numba has to a +heap object per node; allocation and release go through numba's NRT reference counting, like the +CPython program. (An index-based tree in preallocated arrays would be faster but changes the +allocation pattern into an arena, so it is not used.) + +Semantics that differ from CPython: counters are int64, recursion uses the native stack (no +RecursionError). The module must be registered in sys.modules before it runs (run.py does). +""" +from numba import deferred_type, int64, njit, optional +from numba.core.types import UniTuple +from numba.experimental import jitclass + +node_type = deferred_type() + + +@jitclass([("left", optional(node_type)), ("right", optional(node_type))]) +class Node: + def __init__(self, left, right): + self.left = left + self.right = right + + +NodeT = Node.class_type.instance_type +node_type.define(NodeT) + +# cache=True is not usable here: numba raises AttributeError: 'Optional' object has no attribute +# 'type' while hashing the recursive jitclass type for the cache index. So this benchmark has no +# cached run; every process compiles (at import, see below). +OPTS = dict(cache=False) + + +# Explicit signatures: numba cannot infer the return type of the self-recursive make() through +# the jitclass's deferred Optional type (AttributeError: 'Optional' object has no attribute +# 'type'). The consequence is that these two functions compile when this module is imported, not +# on the first call; run.py counts the import time as compile time. +@njit(NodeT(int64), **OPTS) +def make(depth): + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +@njit(int64(optional(NodeT)), **OPTS) +def check(node): + left = node.left + right = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +@njit(UniTuple(int64, 3)(int64), **OPTS) +def kernel(n): + min_depth = 4 + max_depth = max(min_depth + 2, n) + stretch = check(make(max_depth + 1)) + long_lived = make(max_depth) + total = 0 + for depth in range(min_depth, max_depth + 1, 2): + iterations = 1 << (max_depth - depth + min_depth) + chk = 0 + for _ in range(iterations): + chk += check(make(depth)) + total += chk + return stretch, total, check(long_lived) + + +def run(n: int) -> str: + stretch, total, long_lived = kernel(n) + return f"{stretch} {total} {long_lived}" diff --git a/benchmarks/typedpython/numba/fannkuch.py b/benchmarks/typedpython/numba/fannkuch.py new file mode 100644 index 000000000..5f21f092e --- /dev/null +++ b/benchmarks/typedpython/numba/fannkuch.py @@ -0,0 +1,62 @@ +"""numba @njit version of py/fannkuch.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: int64 numpy arrays and int64 counters (wrap instead of +growing), no bounds checks. +""" +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def fannkuch(n): + perm1 = np.arange(n) + count = np.zeros(n, dtype=np.int64) + perm = np.zeros(n, dtype=np.int64) + max_flips = 0 + checksum = 0 + perm_count = 0 + r = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + for i in range(n): + perm[i] = perm1[i] + flips = 0 + k = perm[0] + while k != 0: + lo = 0 + hi = k + while lo < hi: + t = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0 = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def run(n: int) -> str: + checksum, max_flips = fannkuch(n) + return f"{checksum} {max_flips}" diff --git a/benchmarks/typedpython/numba/nbody.py b/benchmarks/typedpython/numba/nbody.py new file mode 100644 index 000000000..74d8b8bee --- /dev/null +++ b/benchmarks/typedpython/numba/nbody.py @@ -0,0 +1,82 @@ +"""numba @njit version of py/nbody.py. Loaded by run.py; `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays instead of lists; error_model="numpy" +(float division by zero gives inf instead of raising ZeroDivisionError); no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +PI = 3.141592653589793 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def offset_momentum(vx, vy, vz, mass): + px = 0.0 + py = 0.0 + pz = 0.0 + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +@njit(**OPTS) +def advance(dt, steps, x, y, z, vx, vy, vz, mass): + for _ in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + d2 = dx * dx + dy * dy + dz * dz + mag = dt / (d2 * math.sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +@njit(**OPTS) +def energy(x, y, z, vx, vy, vz, mass): + e = 0.0 + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx = x[i] - x[j] + dy = y[i] - y[j] + dz = z[i] - z[j] + e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def run(n: int) -> str: + x = np.array([0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01]) + y = np.array([0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01]) + z = np.array([0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01]) + vx = np.array([0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR]) + vy = np.array([0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR]) + vz = np.array([0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR]) + mass = np.array([SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS]) + offset_momentum(vx, vy, vz, mass) + before = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + after = energy(x, y, z, vx, vy, vz, mass) + return f"{before:.9f} {after:.9f}" diff --git a/benchmarks/typedpython/numba/spectral_norm.py b/benchmarks/typedpython/numba/spectral_norm.py new file mode 100644 index 000000000..bfa99bae5 --- /dev/null +++ b/benchmarks/typedpython/numba/spectral_norm.py @@ -0,0 +1,61 @@ +"""numba @njit version of py/spectral_norm.py. `run(n) -> str` is the whole benchmark. + +Semantics that differ from CPython: float64 numpy arrays, int64 index arithmetic (wraps), +error_model="numpy" (no ZeroDivisionError), no bounds checks. +""" +import math + +import numpy as np +from numba import njit + +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def eval_a(i, j): + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +@njit(**OPTS) +def mul_av(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atv(n, v, out): + for i in range(n): + s = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +@njit(**OPTS) +def mul_atav(n, v, out, tmp): + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +@njit(**OPTS) +def kernel(n): + u = np.ones(n) + v = np.zeros(n) + tmp = np.zeros(n) + for _ in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + vbv = 0.0 + vv = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + return vbv, vv + + +def run(n: int) -> str: + vbv, vv = kernel(n) + return f"{math.sqrt(vbv / vv):.9f}" diff --git a/benchmarks/typedpython/numba/wordfreq.py b/benchmarks/typedpython/numba/wordfreq.py new file mode 100644 index 000000000..5bc8599fb --- /dev/null +++ b/benchmarks/typedpython/numba/wordfreq.py @@ -0,0 +1,69 @@ +"""numba @njit version of py/wordfreq.py: numba.typed.Dict[unicode_type, int64]. + +Semantics that differ from CPython: int64 arithmetic (wraps), words are numba unicode strings, +the counts dict is a numba typed.Dict (hash order differs, but the result does not depend on +iteration order because ties are broken by string comparison). +""" +import numpy as np +from numba import njit, types +from numba.typed import Dict, List + +VOCAB = 50000 +MOD = 2147483647 +OPTS = dict(cache=True, error_model="numpy") + + +@njit(**OPTS) +def next_state(s): + return (s * 48271) % MOD + + +@njit(**OPTS) +def word_of(word_id): + x = word_id + 17576 + w = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w + + +@njit(**OPTS) +def kernel(n): + counts = Dict.empty(key_type=types.unicode_type, value_type=types.int64) + s = 12345 + for _ in range(n): + s = next_state(s) + a = s % VOCAB + s = next_state(s) + b = s % VOCAB + word = word_of(a * b // VOCAB) + if word in counts: + counts[word] += 1 + else: + counts[word] = 1 + weighted = 0 + for w in counts: + c = counts[w] + weighted += c * c + taken = List.empty_list(types.unicode_type) + best_counts = np.zeros(5, dtype=np.int64) + for k in range(5): + best_word = "" + best_count = -1 + for w in counts: + c = counts[w] + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + best_counts[k] = best_count + return len(counts), weighted, taken, best_counts + + +def run(n: int) -> str: + size, weighted, taken, best_counts = kernel(n) + parts = [f"{w}:{c}" for w, c in zip(taken, best_counts)] + return f"{size} {weighted} {','.join(parts)}" diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index c7fced567..b6ce8a280 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -13,6 +13,27 @@ the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. +Cython and numba columns (issue: compare TypedPython fairly against them). They need a harness venv +that holds Cython, numba and a C compiler (Xcode command line tools), created ONLY with uv: + + uv venv --python 3.14 benchmarks/typedpython/.venv + uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba + +`--tools-python` (default benchmarks/typedpython/.venv/bin/python) names that interpreter; without it +the three columns report n/a. Columns, each in the form most favourable to it: + "Cython (pure)" py/*.py compiled unmodified by cythonize (annotations as written), same CPython; + "Cython (typed)" cython/*.pyx: cdef types, C arrays, boundscheck/wraparound off, cdef class Node; + "numba" numba/*.py with @njit(cache=True) (binary-trees: jitclass, no cache). The time + reported is the STEADY-STATE call at benchmark size, inside the process, after a + warm-up call that triggered the JIT. The JIT cost is the separate record + "numba compile" (first call: cold cache vs cached; numba import excluded). +Cython is built by cython/build.py before any timing (build/cython/, git-ignored). numba runs in a +fresh process with its own NUMBA_CACHE_DIR (build/numba-cache/, wiped per cold run). +"CPython (tools python)" is the same program under the tools interpreter, and is the baseline for the +Cython/numba ratios in `interleaved` (so those pairs use one interpreter); it reads "same as CPython" +when --python and --tools-python are the same version. Per-column tool version, flags and a semantics +note are written to the JSON ("tools") and printed after the table. + binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The speedup is reported against that CPython version; `CPython (no __slots__)` runs the same program with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. @@ -21,6 +42,7 @@ import datetime import json import os +import shutil import socket import statistics import subprocess @@ -45,7 +67,31 @@ "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), } NOSLOTS = "CPython (no __slots__)" -COLUMNS = ["CPython", NOSLOTS, "Node", "Java", "Rust", "TypedPython @compiled"] +CPY_TOOLS = "CPython (tools python)" +CY_PURE = "Cython (pure)" +CY_TYPED = "Cython (typed)" +NUMBA = "numba" +TOOL_COLS = [CY_PURE, CY_TYPED, NUMBA] +COLUMNS = ["CPython", NOSLOTS, CPY_TOOLS, "Node", "Java", "Rust", "TypedPython @compiled", *TOOL_COLS] +TOOLS_PY_DEFAULT = ROOT / ".venv" / "bin" / "python" +CYTHON_OUT = ROOT / "build" / "cython" +NUMBA_CACHE = ROOT / "build" / "numba-cache" +# tiny sizes for numba's warm-up (first) call: the types are the same at any size, so the first +# call is almost all compile time +NUMBA_WARM = {"nbody": "10", "spectral-norm": "4", "fannkuch-redux": "4", "binary-trees": "1", + "wordfreq": "10"} +CYTHON_FLAGS = "cython/build.py: cythonize(language_level=3), cc -O3 -ffp-contract=off (no fused multiply-add)" +SEMANTICS = { + CY_PURE: "py/*.py unmodified; Cython honours the existing annotations (float is a C double, int stays " + "a Python int, list stays a list), so semantics are CPython's apart from float arithmetic on typed locals", + CY_TYPED: "C semantics where declared: C long/int arithmetic wraps instead of growing, cdivision (no " + "ZeroDivisionError), boundscheck/wraparound off on C arrays, cdef class Node (no __dict__), " + "C-stack recursion (no RecursionError); wordfreq keeps a Python dict", + NUMBA: "fixed-width int64/float64 (ints wrap, no Python big ints, no OverflowError), error_model=numpy " + "(no ZeroDivisionError), no bounds checks, numpy arrays for lists, typed.Dict/List for dict/list " + "(wordfreq), jitclass for the tree (binary-trees); reported time is steady state, JIT is separate", +} + SLOTS_LINE = ' __slots__ = ("left", "right")\n' NOSLOTS_BENCHMARKS = {"binary-trees"} NOT_BUILT = "n/a (not built)" @@ -118,8 +164,64 @@ def noslots_source(name: str) -> Path: return out +TOOLS_PROBE = """ +import json, sys +out = {"python": sys.version.split()[0]} +for mod in ("Cython", "numba", "llvmlite", "numpy", "setuptools"): + try: + out[mod] = __import__(mod).__version__ + except Exception as e: + out[mod] = f"missing ({type(e).__name__}: {e})" +print(json.dumps(out)) +""" + + +def _cython_up_to_date() -> bool: + sources = [*(ROOT / "py").glob("*.py"), *(ROOT / "cython").glob("*"), ] + newest = max(p.stat().st_mtime for p in sources) + for name, (py, *_r) in BENCHMARKS.items(): + stem = Path(py).stem + for kind, mod in (("pure", stem), ("typed", stem + "_typed")): + found = list((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if not found or found[0].stat().st_mtime < newest: + return False + return True + + +def build_tools(tools_python: str | None, python: str) -> dict[str, object]: + """Probe the tools interpreter and cythonize + C-compile the Cython columns (never in a timed run). + Returns {"python", "versions", "na": {column: reason}, "same_python"}.""" + tools: dict[str, object] = {"python": None, "versions": {}, "na": {}, "same_python": True} + na: dict[str, str] = tools["na"] # type: ignore[assignment] + if not tools_python or not Path(tools_python).exists(): + why = ("n/a (no harness venv; create it with: uv venv --python 3.14 benchmarks/typedpython/.venv && " + "uv pip install --python benchmarks/typedpython/.venv/bin/python cython setuptools numba)") + return {**tools, "na": {c: why for c in TOOL_COLS}} + probe = subprocess.run([tools_python, "-c", TOOLS_PROBE], capture_output=True, text=True) + if probe.returncode != 0: + return {**tools, "na": {c: f"n/a (tools python failed: {probe.stderr.strip()[-200:]})" for c in TOOL_COLS}} + versions = json.loads(probe.stdout.strip().splitlines()[-1]) + tools.update(python=tools_python, versions=versions) + base = subprocess.run([python, "-c", "import sys; print(sys.version.split()[0])"], + capture_output=True, text=True).stdout.strip() + tools["same_python"] = base == versions["python"] + tools["baseline_python"] = base + if "missing" in versions["Cython"] or "missing" in versions["setuptools"]: + na[CY_PURE] = na[CY_TYPED] = f"n/a (Cython {versions['Cython']}; setuptools {versions['setuptools']})" + elif not _cython_up_to_date(): + build = subprocess.run([tools_python, str(ROOT / "cython" / "build.py"), str(CYTHON_OUT)], + capture_output=True, text=True) + if build.returncode != 0: + reason = f"n/a (cython build failed: {' | '.join(build.stderr.strip().splitlines()[-3:])})" + na[CY_PURE] = na[CY_TYPED] = reason + if "missing" in versions["numba"] or "missing" in versions["numpy"]: + na[NUMBA] = f"n/a (numba {versions['numba']}; numpy {versions['numpy']})" + return tools + + def commands(name: str, size: str, python: str, java_source: bool, - compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: + compiled: dict[str, tuple[str, list[str]] | str] | None = None, + tools: dict[str, object] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ else ["java", "-cp", str(JAVA_OUT), cls, size] @@ -134,37 +236,188 @@ def commands(name: str, size: str, python: str, java_source: bool, entry = (compiled or {}).get(name) if isinstance(entry, tuple): cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] + if tools and tools.get("python"): + tp = str(tools["python"]) + na: dict[str, str] = tools["na"] # type: ignore[assignment] + stem = Path(py).stem + cmds[CPY_TOOLS] = Cmd([tp, str(ROOT / "py" / py), size]) + for col, kind, mod in ((CY_PURE, "pure", stem), (CY_TYPED, "typed", stem + "_typed")): + so = sorted((CYTHON_OUT / kind).glob(f"{mod}.*.so")) + if col not in na and so: + cmd = Cmd([tp, "-c", RUN_SNIPPET, mod, str(so[0]), size]) + cmd.soft = True + cmds[col] = cmd + if NUMBA not in na: + cmd = Cmd([tp, "-c", NUMBA_SNIPPET, str(ROOT / "numba" / py), stem + "_numba", + NUMBA_WARM[name], size]) + cmd.numba = True + cmd.soft = True + cmd.cache_dir = NUMBA_CACHE / name + cmd.env = {"NUMBA_CACHE_DIR": str(cmd.cache_dir)} + cmds[NUMBA] = cmd return cmds -def run_once(cmd: list[str]) -> tuple[float, str]: +class RunFailed(Exception): + """A Cython/numba column failed to run; reported as n/a with the concrete error.""" + + +class Cmd(list): + """argv plus, for the numba column, an environment, a cache directory and the last timing.""" + env: dict[str, str] | None = None + numba = False + cache_dir: Path | None = None + cold = True + last: dict[str, float] | None = None + soft = False + + +NUMBA_SNIPPET = """ +import importlib.util, json, sys, time +import numba, numpy # imported first: import time is not compile time +path, name, warm, size = sys.argv[1:5] +t0 = time.perf_counter() +spec = importlib.util.spec_from_file_location(name, path) +m = importlib.util.module_from_spec(spec) +sys.modules[name] = m # jitclass resolves its module through sys.modules +spec.loader.exec_module(m) # binary-trees (explicit signatures) compiles here +m.run(int(warm)) # first call: JIT compile (or cache load) at a tiny size +t1 = time.perf_counter() +out = m.run(int(size)) # steady state: already compiled +t2 = time.perf_counter() +print(out) +print("#timing " + json.dumps({"first_call_s": t1 - t0, "steady_s": t2 - t1})) +""" + + +def run_once(cmd: list[str], soft: bool = False) -> tuple[float, str]: + """Wall time and stdout. A numba Cmd returns its steady-state seconds instead (see NUMBA_SNIPPET).""" + cache_dir = getattr(cmd, "cache_dir", None) + if cache_dir is not None and getattr(cmd, "cold", True): + shutil.rmtree(cache_dir, ignore_errors=True) + env = dict(os.environ, **cmd.env) if getattr(cmd, "env", None) else None # type: ignore[attr-defined] start = time.perf_counter() - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmd, capture_output=True, text=True, env=env) elapsed = time.perf_counter() - start if result.returncode != 0: + if soft or getattr(cmd, "soft", False): + last = (result.stderr.strip().splitlines() or ["no stderr"])[-1] + raise RunFailed(f"exit {result.returncode}: {last}") sys.stderr.write(f"failed ({result.returncode}): {' '.join(cmd)}\n{result.stderr}") sys.exit(1) - return elapsed, result.stdout.strip() + out = result.stdout.strip() + if getattr(cmd, "numba", False): + lines = out.splitlines() + timing = json.loads(lines[-1][len("#timing "):]) + cmd.last = dict(timing, wall_s=elapsed) # type: ignore[attr-defined] + out = "\n".join(lines[:-1]).strip() + elapsed = timing["steady_s"] + return elapsed, out + + +def tools_for(args: argparse.Namespace) -> dict[str, object]: + return build_tools(args.tools_python, args.python) + + +def tools_report(tools: dict[str, object]) -> dict[str, object]: + """Per-column tool version, flags and semantics note, for the printout and the JSON.""" + v: dict[str, str] = tools.get("versions") or {} # type: ignore[assignment] + na: dict[str, str] = tools["na"] # type: ignore[assignment] + py = f"CPython {v.get('python', '?')}" + report: dict[str, object] = { + "tools_python": {"interpreter": py, "baseline_python": tools.get("baseline_python"), + "same_interpreter_as_CPython_column": tools.get("same_python")}, + CY_PURE: {"tool": f"Cython {v.get('Cython')} on {py}", "flags": CYTHON_FLAGS, + "semantics": SEMANTICS[CY_PURE]}, + CY_TYPED: {"tool": f"Cython {v.get('Cython')} on {py}", + "flags": CYTHON_FLAGS + "; # cython: boundscheck=False, wraparound=False, cdivision=True, " + "initializedcheck=False in each .pyx", "semantics": SEMANTICS[CY_TYPED]}, + NUMBA: {"tool": f"numba {v.get('numba')}, llvmlite {v.get('llvmlite')}, numpy {v.get('numpy')} on {py}", + "flags": "@njit(cache=True, error_model='numpy'); binary-trees: jitclass + explicit " + "signatures, cache=False; NUMBA_CACHE_DIR per benchmark", + "semantics": SEMANTICS[NUMBA]}, + } + for col in TOOL_COLS: + if col in na: + report[col]["unavailable"] = na[col] # type: ignore[index] + return report + + +def print_tools_report(report: dict[str, object]) -> None: + print("\ntools:") + for key, val in report.items(): + if key == "tools_python": + print(f" tools python: {val['interpreter']}; baseline CPython column: {val['baseline_python']}; " # type: ignore[index] + f"same interpreter: {val['same_interpreter_as_CPython_column']}") # type: ignore[index] + else: + print(f" {key}: {val['tool']}\n flags: {val['flags']}\n semantics: {val['semantics']}") # type: ignore[index] + + +def run_column(col: str, cmd: list[str]) -> tuple[float, str] | str: + """run_once, but a Cython/numba failure becomes the n/a string instead of ending the run.""" + try: + return run_once(cmd) + except RunFailed as e: + return f"n/a ({e})" def verify(args: argparse.Namespace) -> int: build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) bad = 0 for name, spec in BENCHMARKS.items(): - outputs = {col: run_once(cmd)[1] - for col, cmd in commands(name, spec[4], args.python, True, compiled).items()} + outputs: dict[str, str] = {} + notes: dict[str, str] = dict(tools["na"]) # type: ignore[arg-type] + for col, cmd in commands(name, spec[4], args.python, True, compiled, tools).items(): + res = run_column(col, cmd) + if isinstance(res, str): + notes[col] = res + else: + outputs[col] = res[1] distinct = set(outputs.values()) status = "OK " if len(distinct) == 1 else "FAIL" print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") + print(f" columns: {', '.join(outputs)}") + for col, why in notes.items(): + print(f" {col}: {why}") if len(distinct) != 1: bad += 1 for col, out in outputs.items(): print(f" {col}: {out}") print("all outputs agree" if bad == 0 else f"{bad} benchmark(s) disagree") + print_tools_report(tools_report(tools)) return 1 if bad else 0 +def measure_numba(cmd: Cmd, repeat: int) -> tuple[dict[str, object], dict[str, object]]: + """Cold-cache runs (each wipes the cache dir, so each compiles) give the steady-state time and the + cold compile time; one more run with the cache left in place gives the cached first-call time.""" + steady, outs, cold_first, walls = [], set(), [], [] + for _ in range(repeat): + t, out = run_once(cmd) + steady.append(t) + outs.add(out) + cold_first.append(cmd.last["first_call_s"]) # type: ignore[index] + walls.append(cmd.last["wall_s"]) # type: ignore[index] + cache_files = len([p for p in cmd.cache_dir.rglob("*") if p.is_file()]) if cmd.cache_dir else 0 + cmd.cold = False + try: + t_c, out_c = run_once(cmd) + outs.add(out_c) + cached_first = cmd.last["first_call_s"] # type: ignore[index] + cached_steady = t_c + finally: + cmd.cold = True + column = {"median_s": statistics.median(steady), "runs_s": steady, "output": sorted(outs), + "process_wall_s_runs": walls, "note": "steady-state call at benchmark size, JIT excluded"} + compile_rec = {"cold_first_call_s_median": statistics.median(cold_first), "cold_first_call_s_runs": cold_first, + "cached_first_call_s": cached_first, "cached_steady_s": cached_steady, + "cache_files_written": cache_files, "cache_used": cache_files > 0, + "note": "first call = module import + first run() at a tiny size; numba import excluded"} + return column, compile_rec + + def measure(args: argparse.Namespace) -> int: load = os.getloadavg() print(f"load average: {load[0]:.2f} {load[1]:.2f} {load[2]:.2f}") @@ -174,6 +427,8 @@ def measure(args: argparse.Namespace) -> int: return 2 build() compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None + tools = tools_for(args) + report = tools_report(tools) results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): size = args.size.get(name, spec[5]) @@ -181,15 +436,29 @@ def measure(args: argparse.Namespace) -> int: entry = (compiled or {}).get(name, NOT_BUILT) if isinstance(entry, tuple): row["compiled_functions"] = entry[1] - for col, cmd in commands(name, size, args.python, False, compiled).items(): - runs = [run_once(cmd) for _ in range(args.repeat)] - outs = {out for _, out in runs} - row[col] = {"median_s": statistics.median(t for t, _ in runs), - "runs_s": [t for t, _ in runs], "output": sorted(outs)} + for col, cmd in commands(name, size, args.python, False, compiled, tools).items(): + if col == CPY_TOOLS and tools.get("same_python"): + continue + try: + if col == NUMBA: + row[col], row["numba compile"] = measure_numba(cmd, args.repeat) # type: ignore[arg-type] + outs = set(row[col]["output"]) # type: ignore[index] + else: + runs = [run_once(cmd) for _ in range(args.repeat)] + outs = {out for _, out in runs} + row[col] = {"median_s": statistics.median(t for t, _ in runs), + "runs_s": [t for t, _ in runs], "output": sorted(outs)} + except RunFailed as e: + row[col] = f"n/a ({e})" + continue if len(outs) != 1: print(f"warning: {name}/{col} produced differing outputs across runs") if not isinstance(entry, tuple): row["TypedPython @compiled"] = entry + for col, why in tools["na"].items(): # type: ignore[union-attr] + row[col] = why + if tools.get("same_python"): + row[CPY_TOOLS] = "same as CPython" measured = [c for c in COLUMNS if isinstance(row.get(c), dict)] outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index] row["outputs_agree"] = len(outs_all) == 1 @@ -206,6 +475,13 @@ def measure(args: argparse.Namespace) -> int: flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") + print("numba column = steady-state call inside the process (JIT excluded); its compile cost:") + for name, row in results.items(): + rec = row.get("numba compile") + if isinstance(rec, dict): + print(f" {name}: cold first call {rec['cold_first_call_s_median']:.3f} s, cached first call " + f"{rec['cached_first_call_s']:.3f} s, cache files written {rec['cache_files_written']}") + print_tools_report(report) if args.write: out_dir = ROOT / "results" @@ -214,7 +490,7 @@ def measure(args: argparse.Namespace) -> int: payload = {"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "load_average": load, "repeat": args.repeat, "python": subprocess.run([args.python, "--version"], capture_output=True, text=True).stdout.strip(), - "results": results} + "tools": report, "results": results} path.write_text(json.dumps(payload, indent=2) + "\n") print(f"wrote {path}") return 0 @@ -224,46 +500,48 @@ def interleaved(args: argparse.Namespace) -> int: """CPython vs TypedPython @compiled, alternated A/B in the same time window (leader decision, 2026-10-03): when the machine is never quiet, load hits both sides of each pair alike, so the per-pair ratio is stable even though absolute times are not. Reports the median ratio, never - absolute times as the result, with the load average recorded at every pair.""" - if not args.typedpython: - print("interleaved needs --typedpython") + absolute times as the result, with the load average recorded at every pair. + + The Cython and numba columns are paired the same way against the CPython run under the tools + interpreter (`CPython (tools python)`), each pair alternating which side runs first. For numba the + ratio uses its in-process steady-state time (JIT excluded, process start included for CPython, a + small bias in numba's favour at these sizes); its first-call (compile) time is recorded per pair.""" + tools = tools_for(args) + if not args.typedpython and not tools.get("python"): + print("interleaved needs --typedpython or a tools python") return 2 - compiled = build_compiled(Path(args.typedpython), args.typedpython_python) + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else {} results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): entry = compiled.get(name) - if not isinstance(entry, tuple): - results[name] = {"skipped": entry} - continue size = args.size.get(name, spec[5]) - cmds = commands(name, size, args.python, False, compiled) - a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] - c_cmd = cmds.get(NOSLOTS) - pairs = [] - for i in range(args.pairs): - order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first - load = os.getloadavg() - (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) - t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) - if o_a != o_b: - print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") - return 1 - pair = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} - if c_cmd is not None: - t_c, o_c = run_once(c_cmd) - if o_c != o_a: - print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") - return 1 - pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) - pairs.append(pair) - ratios = [p["ratio"] for p in pairs] - results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, - "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), - "max_ratio": max(ratios)} - if c_cmd is not None: - results[name]["median_ratio_vs_noslots"] = statistics.median( - p["ratio_vs_noslots"] for p in pairs) + cmds = commands(name, size, args.python, False, compiled, tools) + r: dict[str, object] = {"size": size} + if args.typedpython: + if isinstance(entry, tuple): + r.update(measure_pair(name, cmds["CPython"], cmds["TypedPython @compiled"], cmds.get(NOSLOTS), + args.pairs, "compiled")) + r["compiled_functions"] = entry[1] + else: + r["skipped"] = entry + extras: dict[str, object] = {} + base = cmds.get(CPY_TOOLS) + for col in TOOL_COLS: + if col in tools["na"]: # type: ignore[operator] + extras[col] = tools["na"][col] # type: ignore[index] + elif col not in cmds or base is None: + extras[col] = "n/a (not built)" + else: + try: + extras[col] = measure_pair(name, base, cmds[col], None, args.pairs, col) + except RunFailed as e: + extras[col] = f"n/a ({e})" + r["tools_pairs"] = extras + results[name] = r print(f"measured {name}", file=sys.stderr) + if any(isinstance(v, dict) and v.get("outputs_differ") for v in [r, *extras.values()]): + print(f"{name}: OUTPUTS DIFFER, see the JSON/table") + return 1 print() print("| benchmark | size | speedup vs CPython (median of pairs) | range | load during pairs (1-min) |") @@ -271,22 +549,69 @@ def interleaved(args: argparse.Namespace) -> int: for name, r in results.items(): if "skipped" in r: print(f"| {name} | - | {r['skipped']} | | |") - continue - loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] - print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}–{r['max_ratio']:.1f}x " - f"| {min(loads):.1f}–{max(loads):.1f} |") - print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + elif "median_ratio" in r: + loads = [p["load"][0] for p in r["pairs"]] # type: ignore[index] + print(f"| {name} | {r['size']} | {r['median_ratio']:.1f}x | {r['min_ratio']:.1f}\u2013{r['max_ratio']:.1f}x " + f"| {min(loads):.1f}\u2013{max(loads):.1f} |") + if args.typedpython: + print(f"\n{args.pairs} A/B pairs per benchmark, order alternated; ratio = CPython time / compiled time.") + print("\n| benchmark | column | speedup vs CPython (tools python), median of pairs | range | extra |") + print("|---|---|---|---|---|") + for name, r in results.items(): + for col, p in r["tools_pairs"].items(): # type: ignore[union-attr] + if isinstance(p, dict): + extra = "" + if col == NUMBA: + extra = (f"first call (compile) median {statistics.median(q['first_call_s'] for q in p['pairs']):.3f} s; " + f"process wall median {statistics.median(q['wall_s'] for q in p['pairs']):.3f} s") + print(f"| {name} | {col} | {p['median_ratio']:.1f}x | {p['min_ratio']:.1f}\u2013{p['max_ratio']:.1f}x | {extra} |") + else: + print(f"| {name} | {col} | {p} | | |") + print(f"\n{args.pairs} pairs per column, order alternated; ratio = CPython (tools python) time / column time.") + report = tools_report(tools) + print_tools_report(report) if args.write: out_dir = ROOT / "results" out_dir.mkdir(exist_ok=True) path = out_dir / f"{datetime.date.today().isoformat()}-{socket.gethostname()}-interleaved.json" path.write_text(json.dumps({"date": datetime.datetime.now().isoformat(timespec="seconds"), "host": socket.gethostname(), "method": "interleaved A/B", - "pairs": args.pairs, "results": results}, indent=2) + "\n") + "pairs": args.pairs, "tools": report, "results": results}, indent=2) + "\n") print(f"wrote {path}") return 0 +def measure_pair(name: str, a_cmd: list[str], b_cmd: list[str], c_cmd: list[str] | None, n_pairs: int, + label: str) -> dict[str, object]: + """n_pairs A/B pairs (A = CPython, B = the column); which runs first alternates every pair.""" + pairs = [] + for i in range(n_pairs): + order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first + load = os.getloadavg() + (t1, o1), (t2, o2) = run_once(order[0]), run_once(order[1]) + t_a, o_a, t_b, o_b = (t1, o1, t2, o2) if order[0] is a_cmd else (t2, o2, t1, o1) + if o_a != o_b: + print(f"{name}/{label}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") + return {"outputs_differ": True} + pair: dict[str, object] = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} + last = getattr(b_cmd, "last", None) + if getattr(b_cmd, "numba", False) and last: + pair.update(first_call_s=last["first_call_s"], wall_s=last["wall_s"]) + if c_cmd is not None: + t_c, o_c = run_once(c_cmd) + if o_c != o_a: + print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") + return {"outputs_differ": True} + pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) + pairs.append(pair) + ratios = [p["ratio"] for p in pairs] + out: dict[str, object] = {"pairs": pairs, "median_ratio": statistics.median(ratios), + "min_ratio": min(ratios), "max_ratio": max(ratios)} + if c_cmd is not None: + out["median_ratio_vs_noslots"] = statistics.median(p["ratio_vs_noslots"] for p in pairs) + return out + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) @@ -294,7 +619,7 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) - ab = sub.add_parser("interleaved", help="CPython vs @compiled alternated A/B; reports median ratios") + ab = sub.add_parser("interleaved", help="CPython vs @compiled/Cython/numba alternated A/B; reports median ratios") ab.add_argument("--python", default=sys.executable) ab.add_argument("--pairs", type=int, default=11) ab.add_argument("--write", action="store_true") @@ -304,6 +629,9 @@ def main() -> int: help="directory holding the typedpython package; enables the @compiled column") p.add_argument("--typedpython-python", default=sys.executable, help="interpreter for compiling (needs pyrefly 1.3.2)") + p.add_argument("--tools-python", default=str(TOOLS_PY_DEFAULT), + help="interpreter of the uv venv holding Cython and numba (default: benchmarks/" + "typedpython/.venv/bin/python); enables the Cython and numba columns") m.add_argument("--repeat", type=int, default=5) m.add_argument("--force", action="store_true") m.add_argument("--write", action="store_true") From b159f27bdfa860add9dba75e177dfc4fa469617e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:45:16 +0900 Subject: [PATCH 02/61] TypedPython: decode cl.exe output in the console code page cl.exe writes in the console code page (CP949 on Korean Windows); decoding it as UTF-8 raised UnicodeDecodeError in subprocess's reader thread and would lose a compile error message. --- .../src/main/python/typedpython/cbuild.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py index d12ecea39..2f30a9490 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py @@ -164,7 +164,7 @@ def _find_vcvars() -> Path: f"with C++' workload ({MSVC_COMPONENT}), or run from a 'x64 Native Tools' prompt.") run = subprocess.run( [str(vswhere), "-latest", "-products", "*", "-requires", MSVC_COMPONENT, - "-property", "installationPath"], capture_output=True, text=True) + "-property", "installationPath"], capture_output=True, encoding="oem", errors="replace") root = run.stdout.strip().splitlines()[0].strip() if run.returncode == 0 and run.stdout.strip() else "" if not root: raise CBuildError( @@ -229,7 +229,8 @@ def build(c_source: str, module_name: str, out_dir: Path, runtime_dir: Path | No env = msvc_env() cmd = msvc_command(_msvc_cl(env), c_file, so, include, runtime, flags, gil_disabled=free_threaded(ext_suffix)) - run = subprocess.run(cmd, capture_output=True, text=True, env=env) + # cl.exe writes in the console code page (CP949 on Korean Windows), not UTF-8 + run = subprocess.run(cmd, capture_output=True, encoding="oem", errors="replace", env=env) if run.returncode != 0 or not so.is_file(): # cl.exe reports its errors on stdout raise CBuildError(f"C compiler failed ({run.returncode}): {subprocess.list2cmdline(cmd)}\n" From 2a75e573edcf6671501a2629ab2555297862ff95 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:51:56 +0900 Subject: [PATCH 03/61] TypedPython: publish the package to PyPI by trusted publishing A typedpython-v* tag builds the wheel, checks that it alone compiles the release example, and publishes it. INTENT 1.4 records the maintainer's distribution decision. --- .github/workflows/typedpython-pypi.yml | 78 +++++++++++++++++++++++++ docs/INTENT.md | 5 ++ python-multiplatform-ksp/pyproject.toml | 6 ++ 3 files changed, 89 insertions(+) create mode 100644 .github/workflows/typedpython-pypi.yml diff --git a/.github/workflows/typedpython-pypi.yml b/.github/workflows/typedpython-pypi.yml new file mode 100644 index 000000000..b0f98705d --- /dev/null +++ b/.github/workflows/typedpython-pypi.yml @@ -0,0 +1,78 @@ +name: TypedPython to PyPI + +# Publishes the `typedpython` package (python-multiplatform-ksp/pyproject.toml) to PyPI. The package +# is versioned on its own, so it has its own tag: `typedpython-v`, pushed by the coordinator +# on a commit of main. PyPI trusted publishing: no token is stored; the `pypi` environment and the +# PyPI project's trusted publisher (this repository, this workflow file, environment `pypi`) must match. +# Only a wheel is published: it is pure Python (`py3-none-any`), and an sdist would carry the whole +# Kotlin module next to it. + +on: + push: + tags: + - 'typedpython-v*' + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to build and publish (e.g. typedpython-v0.1.0)' + required: true + +permissions: + contents: read + +jobs: + build: + name: Build and check the wheel + runs-on: macos-latest + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + + - name: The tag names the package version + run: | + set -euo pipefail + version="$(sed -nE 's/^version = "(.*)"$/\1/p' python-multiplatform-ksp/pyproject.toml | head -1)" + test "$TAG" = "typedpython-v$version" || { echo "error: tag $TAG does not match typedpython $version" >&2; exit 1; } + + - name: Build + run: uv build --wheel --out-dir dist python-multiplatform-ksp + + - name: The wheel alone compiles the release example + run: | + set -euo pipefail + uv venv -q -p 3.15t "$RUNNER_TEMP/tpenv" + uv pip install -q --python "$RUNNER_TEMP/tpenv/bin/python" dist/typedpython-*.whl + "$RUNNER_TEMP/tpenv/bin/typedpython" demo --out "$RUNNER_TEMP/tpbuild" docs/release/desktop/typedpython/kernels.py 400000 | tee "$RUNNER_TEMP/tp.log" + grep -q '^compiled functions: .*dot' "$RUNNER_TEMP/tp.log" + grep -q '^identical output' "$RUNNER_TEMP/tp.log" + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + + - name: Publish + run: uv publish --trusted-publishing always dist/* diff --git a/docs/INTENT.md b/docs/INTENT.md index cb707781c..5c45079d1 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -40,6 +40,11 @@ CPython interpreter inside a Kotlin Multiplatform application so that: is **checked** by default; **compiling** it to native code is opt-in. The compiled code still runs on the embedded CPython, as an extension module (`docs/design/typedpython.md`). + TypedPython reaches plain Python users as a **distribution**, not as a new interpreter (maintainer, + 2026-10-30: "배포판 방식을 생각했다는거야. 근데 일단 그렇게 하더라도 typedpython 패키지 형태라면 pypi + 배포를 해야겠지"). The thisisthepy CPython build ships with TypedPython in it, and the same + compiler is an ordinary `typedpython` package on PyPI (`uv add typedpython`). + 5. **High-performance kernels are written in Python too** (TypedPython tier 2). The maintainer first asked to explore it ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자") and on 2026-10-04 made it a commitment ("GPU 커널 등 탐색을 약속으로 올려"), so that TypedPython can stand diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index d5fcbf41c..487554414 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -4,11 +4,17 @@ [project] name = "typedpython" version = "0.1.0" +description = "TypedPython: statically checked Python, with marked functions compiled to CPython extension modules" +license = "Apache-2.0" +authors = [{ name = "thisisthepy" }] requires-python = ">=3.13" # Pinned exactly: the gate reads Pyrefly's per-expression type report (--report-pysa), whose # format is versioned by Pyrefly, not by a public API. dependencies = ["pyrefly==1.3.2"] +[project.urls] +Homepage = "https://github.com/thisisthepy/python-multiplatform" + [project.scripts] typedpython = "typedpython.__main__:cli" From a59096f40bf5cdd7f89d6889b353001518a1db03 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:54:13 +0900 Subject: [PATCH 04/61] TypedPython: @compiled is the only mark; drop the '# typedpython: compiled' module pragma The maintainer decided (2026-10-30) to keep only the builtin @compiled. The pragma is now an ordinary comment; tests, the Kotlin-bridge fixture, benchmarks and the release example decorate the functions the pragma used to cover. --- .../2026-10-17-buffer-atomic-kernels.py | 4 +- benchmarks/typedpython/run.py | 16 +++- docs/SPEC.md | 2 +- docs/design/typedpython.md | 6 +- docs/release/desktop/typedpython/kernels.py | 3 +- .../typedpython/tp_kotlin_bridge.py | 3 +- .../src/main/python/typedpython/__main__.py | 4 +- .../src/main/python/typedpython/frontend.py | 26 ++----- .../src/test/python/conftest.py | 20 +++++ .../src/test/python/test_bitwise.py | 27 ++++++- .../src/test/python/test_buffer_arrays.py | 24 +++++- .../src/test/python/test_buffer_memoryview.py | 11 ++- .../src/test/python/test_buffer_numpy.py | 76 +++++++++++++++++- .../src/test/python/test_class_methods.py | 20 +++-- .../src/test/python/test_cost_rule.py | 19 +++-- .../src/test/python/test_eval_breaker.py | 19 ++++- .../src/test/python/test_fast_entry.py | 13 +++- .../src/test/python/test_free_threaded.py | 20 ++++- .../src/test/python/test_frontend.py | 29 +++++-- .../src/test/python/test_frontend_classes.py | 9 +-- .../src/test/python/test_interval_proofs.py | 17 +++- .../src/test/python/test_module_report.py | 29 +++---- .../src/test/python/test_native_int_impure.py | 35 +++++++-- .../src/test/python/test_osr_exit.py | 4 +- .../src/test/python/test_parallel_loops.py | 78 +++++++++++++++++-- .../src/test/python/test_pipeline.py | 16 ++-- .../test/python/test_pipeline_incremental.py | 19 +++-- .../src/test/python/test_recursion.py | 8 +- .../src/test/python/test_regions.py | 13 +++- .../src/test/python/test_resume.py | 4 +- .../src/test/python/test_str_dict.py | 38 ++++++++- .../src/test/python/test_tracing.py | 5 +- 32 files changed, 498 insertions(+), 119 deletions(-) diff --git a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py index a0cd59c56..53a917186 100644 --- a/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py +++ b/benchmarks/typedpython/results/2026-10-17-buffer-atomic-kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import array +@compiled def dot_arr(a: array.array[float], b: array.array[float]) -> float: s: float = 0.0 for i in range(len(a)): @@ -9,11 +9,13 @@ def dot_arr(a: array.array[float], b: array.array[float]) -> float: return s +@compiled def scale_arr(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k +@compiled def dot_list(a: list[float], b: list[float]) -> float: s: float = 0.0 for i in range(len(a)): diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 104ad572d..c6c3aaa47 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -10,7 +10,7 @@ The "TypedPython @compiled" column runs only with `--typedpython ` (the directory that holds the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with -the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is +`@compiled` above every top-level def and class, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The @@ -18,6 +18,7 @@ with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. """ import argparse +import ast import datetime import json import os @@ -87,6 +88,17 @@ def build() -> None: """ +def with_compiled(source: str) -> str: + """`source` with `@compiled` above every top-level def and class (above their other decorators).""" + tree = ast.parse(source) + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[str]] | str]: """Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason.""" built: dict[str, tuple[str, list[str]] | str] = {} @@ -95,7 +107,7 @@ def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[st work = TP_BUILD / name work.mkdir(parents=True, exist_ok=True) source = work / py - source.write_text("# typedpython: compiled\n" + (ROOT / "py" / py).read_text()) + source.write_text(with_compiled((ROOT / "py" / py).read_text())) result = subprocess.run([tp_python, "-c", COMPILE_SNIPPET, str(source), str(work / "out")], capture_output=True, text=True, env=env) if result.returncode != 0: diff --git a/docs/SPEC.md b/docs/SPEC.md index 1ba12bbc1..61c454c2f 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -724,7 +724,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. with `.diagnostics`) before the module runs. Results are cached in memory by file content hash, failures included. `Status: implemented`, `python-multiplatform-ksp/src/test/python/test_importgate.py`. - **N-8** TypedPython compilation to C (INTENT §1.4; design §4.3; #41). `@compiled` (a builtin name, no - import, #42) and a first-line `# typedpython: compiled` mark functions; the compiler lowers them + import, #42) is the only mark for functions (the first-line module pragma was removed, #353); the compiler lowers them through the typed IR (`typedpython/ir.py`) to C against the CPython C API and builds an extension with the platform C compiler. Compiled functions return what CPython returns (IR contract: checked i64 with deopt to the interpreted function, Python floor division/modulo, exact int/float comparison, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 1ae91c1c2..991207854 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -62,7 +62,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음, 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부, 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` (내장 이름, import 없음) 또는 모듈 선언 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | +| 1 `compiled` | `@compiled` (내장 이름, import 없음) | `checked` + §3.1 금지 목록 | C 확장 모듈 (NEON · AVX 벡터화, §5.3) | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL (직접) / WGSL / PTX, 빌드 타임 (§5) | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -685,9 +685,7 @@ upstream 대비 수치 오차와 커널 시간을 기준선으로 박아 둔다. 인터프리터 시작 시 `builtins.compiled` 를 설치하고(#42), 일반 CPython 에서는 `typedpython` wheel 의 `typedpython_builtins.pth` 가 `builtins` 에 없을 때만 항등 함수를 설치한다(런타임이 넣은 것을 덮어쓰지 않는다). 타입 검사기에는 `__builtins__.pyi` 로 알린다, Pyrefly 1.3.2 는 검색 경로의 그 파일을 추가 - 내장으로 읽는다(실측). 모듈 단위 표기는 `# typedpython: compiled` 주석을 유지한다: `# pyright: strict` 와 같은 - 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 - 모듈 전체를 지정할 수 있다. + 내장으로 읽는다(실측). 모듈 단위 pragma(`# typedpython: compiled`)는 2026-10-30 메인테이너 결정으로 제거했다(#353). `@compiled` 가 컴파일 대상을 표시하는 유일한 수단이다. 2. ~~고정폭 정수의 오버플로 의미~~, **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). 3. ~~pypackpack Cython 슬롯 인터페이스 소유~~, **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, C 를 낸 모듈 목록을 넘긴다. 확장이 원본 소스를 품으므로 별도 `.py` 폴백은 없다(Cython 폐기 후 갱신). 4. ~~GPU 착지를 torchnative 커스텀 op 로 할 것인가~~, **결정됨 (2026-10-04, 메인테이너):** 커널은 diff --git a/docs/release/desktop/typedpython/kernels.py b/docs/release/desktop/typedpython/kernels.py index 81cc9f952..3fb98914e 100644 --- a/docs/release/desktop/typedpython/kernels.py +++ b/docs/release/desktop/typedpython/kernels.py @@ -1,7 +1,7 @@ -# typedpython: compiled import sys +@compiled def dot(n: int) -> float: s = 0.0 for i in range(n): @@ -10,6 +10,7 @@ def dot(n: int) -> float: return s +@compiled def collatz_steps(limit: int) -> int: total = 0 for start in range(1, limit): diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py index 7677eca9c..311eb2f5b 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py @@ -1,4 +1,3 @@ -# typedpython: compiled """Issue #45: compiled TypedPython code calling Kotlin through the binder. Every name imported below is a binder proxy (`fixture/app/TypedPythonBridge.kt`, published by @@ -21,6 +20,7 @@ from fixture.app import TpSample, tpAccumulate, tpCheck, tpTouch +@compiled def drive(n: int, scale: float, token: object) -> float: """Per step: a Kotlin constructor call that returns a typed proxy, a method call on that proxy, the proxy handed back to a Kotlin function, and a PyObject handed to Kotlin -- inside native @@ -35,6 +35,7 @@ def drive(n: int, scale: float, token: object) -> float: return total +@compiled def checked(n: int, limit: float) -> float: """`tpCheck` throws in Kotlin once `acc` passes `limit`; the caller sees what Python sees.""" acc: float = 0.0 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index bd220e713..a9cda63e2 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -84,7 +84,7 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: print(f"{m.status:7} {m.name}: {what}{why}") rel = m.path.relative_to(root).as_posix() if m.kind == "not_marked": - print(f" {rel}: not marked for compilation (no `# typedpython: compiled`, no @compiled)") + print(f" {rel}: not marked for compilation (no @compiled)") for item in m.skipped_items: where = f"{item.file}:{item.line}" if item.line is not None else item.file print(f" {where}: {item.name}: {item.message}") @@ -112,6 +112,8 @@ def _demo(module: Path, module_argv: list[str], out: Path) -> int: return EXIT_FAILURE def interpreted() -> object: + import builtins # the module's `@compiled` is a builtin name; typedpython_builtins.pth installs it + builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 0da894f10..6e0b75aa6 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -3,8 +3,7 @@ `lower(path)` reads one module and returns an `ir.Module`: - Which functions: module-level `def`s decorated with the builtin `@compiled` (a bare name, no - import, #42), or every module-level `def` when the first non-empty line is - `# typedpython: compiled`. + import, #42). That decorator is the only mark; there is no module-level pragma (#353). - A function is lowered only when **all** of it is expressible in the IR; otherwise it is left out and `Module.skipped[name]` says why, with the line. There is no partial lowering. - Types: parameters and returns from their annotations (`int`/`float`/`bool`, `list[float]` and @@ -80,8 +79,6 @@ from typedpython import gate, ir, pyrefly, rebinding, resume, taint from typedpython.ir import Type -MARKER = "# typedpython: compiled" - I64_MIN, I64_MAX = -2**63, 2**63 - 1 TOP = (I64_MIN, I64_MAX) EXACT_INT_FLOAT = 2**53 # |n| <= this: int/int true division is exact-rounded in C (ir.BinOp) @@ -249,7 +246,7 @@ def lower(path: Path) -> ir.Module: def _parallel_marked(tree: ast.Module, source: str) -> set[str]: """Module-level functions the user marked for SPEC N-15: the comment line `# typedpython: parallel` - directly above the `def` (above its decorators if it has any). A comment, like the module marker, + directly above the `def` (above its decorators if it has any). A comment, so the interpreted module is unchanged and needs no runtime name.""" lines = source.splitlines() out: set[str] = set() @@ -434,17 +431,10 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") -def _whole_module(source: str) -> bool: - return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER - - def is_opted_in(path: Path) -> bool: - """Whether the front end considers the module for compilation at all: the `# typedpython: compiled` - marker as its first non-empty line, or a module-level `def` / `class` decorated `@compiled` - (the same tests `_targets` and `_classes` apply).""" + """Whether the front end considers the module for compilation at all: a module-level `def` / `class` + decorated `@compiled` (the same test `_targets` and `_classes` apply).""" source = Path(path).read_text() - if _whole_module(source): - return True return any( isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) @@ -454,14 +444,13 @@ def is_opted_in(path: Path) -> bool: def _classes( tree: ast.Module, source: str, info: _ModuleInfo, ) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]: - whole = _whole_module(source) decls: dict[str, ir.ClassDecl] = {} skipped: dict[str, str] = {} for node in tree.body: if not isinstance(node, ast.ClassDef): continue marked = any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) - if not (whole or marked): + if not marked: continue result = _class_decl(node, tree, info) if isinstance(result, str): @@ -608,13 +597,12 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, ) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - whole_module = _whole_module(source) found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] - if not (whole_module or marked): + if not marked: continue reason = None if marked and info.bound("compiled"): @@ -2499,4 +2487,4 @@ def _breaks(body: tuple[ir.Stmt, ...]) -> bool: return False -__all__ = ["lower", "MARKER"] +__all__ = ["lower"] diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 452a4e613..669816bf4 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -1,12 +1,32 @@ +import ast +import builtins import sys from pathlib import Path import textwrap import pytest +# The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. +builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) + FIXTURES = Path(__file__).parent / "fixtures" +def decorate_compiled(source: str) -> str: + """`source` with `@compiled` put above every module-level def / class (above its other decorators), + the set a whole-module mark used to cover. A source that does not parse is returned unchanged.""" + try: + tree = ast.parse(source) + except SyntaxError: + return source + lines = source.split("\n") + firsts = [min([n.lineno] + [d.lineno for d in n.decorator_list]) for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))] + for first in reversed(firsts): + lines.insert(first - 1, "@compiled") + return "\n".join(lines) + + @pytest.fixture def write(tmp_path): """Write a dedented source file under tmp_path and return its path.""" diff --git a/python-multiplatform-ksp/src/test/python/test_bitwise.py b/python-multiplatform-ksp/src/test/python/test_bitwise.py index 0bd844034..c0d48b3d4 100644 --- a/python-multiplatform-ksp/src/test/python/test_bitwise.py +++ b/python-multiplatform-ksp/src/test/python/test_bitwise.py @@ -20,33 +20,39 @@ from test_native_int_impure import build_source SOURCE = textwrap.dedent(''' - # typedpython: compiled + @compiled def p_and(a: int, b: int) -> int: return a & b + @compiled def p_or(a: int, b: int) -> int: return a | b + @compiled def p_xor(a: int, b: int) -> int: return a ^ b + @compiled def p_inv(a: int) -> int: return ~a + @compiled def p_shl(a: int, b: int) -> int: return a << b + @compiled def p_shr(a: int, b: int) -> int: return a >> b + @compiled def p_aug(a: int, b: int) -> int: x: int = a x ^= b @@ -55,22 +61,26 @@ def p_aug(a: int, b: int) -> int: return x + @compiled def p_augshl(a: int, b: int) -> int: x: int = a x <<= b return x + @compiled def p_augshr(a: int, b: int) -> int: x: int = a x >>= b return x + @compiled def p_mix(a: int, b: int) -> int: return (((a & 255) << 3) | (b >> 2)) ^ ~a + @compiled def p_mask_loop(n: int) -> int: h: int = 0 for i in range(n): @@ -78,6 +88,7 @@ def p_mask_loop(n: int) -> int: return h + @compiled def p_array(xs: list[int]) -> None: for i in range(len(xs)): xs[i] &= 0xFF @@ -85,10 +96,12 @@ def p_array(xs: list[int]) -> None: xs[i] ^= i + @compiled def p_iv(a: int, b: int) -> int: return (a & 0xFFFF) * (b & 0xFFFF) + ~(a & 7) + @compiled def i_and(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a & b @@ -96,6 +109,7 @@ def i_and(log: list[object], a: int, b: int) -> object: return r + @compiled def i_or(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a | b @@ -103,6 +117,7 @@ def i_or(log: list[object], a: int, b: int) -> object: return r + @compiled def i_xor(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a ^ b @@ -110,6 +125,7 @@ def i_xor(log: list[object], a: int, b: int) -> object: return r + @compiled def i_inv(log: list[object], a: int) -> object: log.append("start") r: int = ~a @@ -117,6 +133,7 @@ def i_inv(log: list[object], a: int) -> object: return r + @compiled def i_shl(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -124,6 +141,7 @@ def i_shl(log: list[object], a: int, b: int) -> object: return r + @compiled def i_shr(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a >> b @@ -131,6 +149,7 @@ def i_shr(log: list[object], a: int, b: int) -> object: return r + @compiled def i_aug(log: list[object], a: int, b: int) -> object: log.append("start") x: int = a @@ -143,6 +162,7 @@ def i_aug(log: list[object], a: int, b: int) -> object: return x + @compiled def i_big(log: list[object], a: int, b: int, c: int) -> object: log.append("start") r: int = (a * b) & c @@ -156,6 +176,7 @@ def i_big(log: list[object], a: int, b: int, c: int) -> object: return r + @compiled def o_ops(log: list[object], x: object, y: object) -> object: log.append("start") r: object = x | y @@ -164,21 +185,25 @@ def o_ops(log: list[object], x: object, y: object) -> object: return r + @compiled def o_inplace(log: list[object], x: object, y: object) -> object: log.append("start") x |= y return x + @compiled def f_and(log: list[object], a: float, b: int) -> object: log.append("start") return a & b + @compiled def b_and(a: bool, b: bool) -> bool: return a & b + @compiled def b_inv(a: bool) -> int: return ~a ''') diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py index 71d95f0c0..724f6759e 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py @@ -29,10 +29,10 @@ from typedpython import cbuild, cgen, frontend, ir, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -40,6 +40,7 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def sum_aq(a: array.array[int]) -> int: s = 0 for i in range(len(a)): @@ -47,70 +48,85 @@ def sum_aq(a: array.array[int]) -> int: return s + @compiled def get_ad(a: array.array[float], i: int) -> float: return a[i] + @compiled def get_aq(a: array.array[int], i: int) -> int: return a[i] + @compiled def set_ad(a: array.array[float], i: int, v: float) -> None: a[i] = v + @compiled def set_aq(a: array.array[int], i: int, v: int) -> None: a[i] = v + @compiled def length_ad(a: array.array[float]) -> int: return len(a) + @compiled def scale_ad(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def iota_aq(a: array.array[int]) -> None: for i in range(len(a)): a[i] = i + @compiled def copy_ad(dst: array.array[float], src: array.array[float]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def cascade_ad(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + 1.0 + @compiled def fill_n(a: array.array[float], n: int) -> None: for i in range(n): a[i] = 1.0 + @compiled def mixed(l: list[float], a: array.array[float]) -> None: for i in range(len(a)): a[i] = l[i] + @compiled def four(a: array.array[float], b: array.array[float], c: array.array[float], d: array.array[float]) -> None: for i in range(len(a)): a[i] = b[i] + c[i] + d[i] + @compiled def div_ad(a: array.array[float]) -> float: return a[0] / a[1] + @compiled def open_ad(a: array.array[float], o: object) -> float: return a[0] + float(o.v) + @compiled def rebind_view(a: array.array[float]) -> None: y = a[:] y[0] = 5.0 @@ -505,30 +521,34 @@ def many(n, buffer): # --- vectorised element-wise store loops (#226) --------------------------------------------------- VEC_SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def vscale(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] * k + @compiled def vmix(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = -(a[i] - k) * 2.5 + k + @compiled def vtwo(a: array.array[float], b: array.array[float]) -> None: for i in range(len(a)): a[i] = a[i] + b[i] + @compiled def vfill(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = k + @compiled def vdiv(a: array.array[float], k: float) -> None: for i in range(len(a)): a[i] = a[i] / k diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index 1766b3300..8a616abc8 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -23,10 +23,10 @@ same_value) SOURCE = textwrap.dedent(''' - # typedpython: compiled import array + @compiled def sum_mf(m: memoryview[float]) -> float: s = 0.0 for i in range(len(m)): @@ -34,6 +34,7 @@ def sum_mf(m: memoryview[float]) -> float: return s + @compiled def sum_mi(m: memoryview[int]) -> int: s = 0 for i in range(len(m)): @@ -41,31 +42,38 @@ def sum_mi(m: memoryview[int]) -> int: return s + @compiled def get_mf(m: memoryview[float], i: int) -> float: return m[i] + @compiled def get_mi(m: memoryview[int], i: int) -> int: return m[i] + @compiled def set_mf(m: memoryview[float], i: int, v: float) -> None: m[i] = v + @compiled def set_mi(m: memoryview[int], i: int, v: int) -> None: m[i] = v + @compiled def length_mf(m: memoryview[float]) -> int: return len(m) + @compiled def scale_mf(m: memoryview[float], k: float) -> None: for i in range(len(m)): m[i] = m[i] * k + @compiled def spin_mf(m: memoryview[float], n: int) -> float: s = 0.0 L = len(m) @@ -74,6 +82,7 @@ def spin_mf(m: memoryview[float], n: int) -> float: return s + @compiled def spin_af(a: array.array[float], n: int) -> float: s = 0.0 L = len(a) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py index 582e4b094..f14a32c00 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_numpy.py @@ -21,13 +21,13 @@ from test_buffer_arrays import build_source, load_so # noqa: E402 SOURCE = textwrap.dedent(''' - # typedpython: compiled import array import math import numpy as np import numpy.typing as npt + @compiled def sum_ad(a: array.array[float]) -> float: s = 0.0 for i in range(len(a)): @@ -35,18 +35,22 @@ def sum_ad(a: array.array[float]) -> float: return s + @compiled def nd_first(x: npt.NDArray[np.float64]) -> object: return x[0] + @compiled def nd_first_f(x: npt.NDArray[np.float64]) -> float: return x[0] + @compiled def nd_first_i(x: npt.NDArray[np.int64]) -> int: return x[0] + @compiled def nd_sum(x: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -54,6 +58,7 @@ def nd_sum(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: s = x[0] for i in range(1, len(x)): @@ -61,6 +66,7 @@ def nd_sum_from_first(x: npt.NDArray[np.float64]) -> float: return s + @compiled def nd_isum(x: npt.NDArray[np.int64]) -> int: s = 0 for i in range(len(x)): @@ -68,64 +74,79 @@ def nd_isum(x: npt.NDArray[np.int64]) -> int: return s + @compiled def nd_gt(x: npt.NDArray[np.float64]) -> object: return x[0] > x[1] + @compiled def nd_flag(x: npt.NDArray[np.float64]) -> object: b = x[0] > 0.0 return b + @compiled def nd_plain(x: npt.NDArray[np.float64]) -> object: return float(x[0]) + @compiled def nd_mixed(x: npt.NDArray[np.float64], k: float) -> object: return k + 1.0 + @compiled def nd_sqrt(x: npt.NDArray[np.float64]) -> object: return math.sqrt(x[0] * x[0]) + @compiled def nd_neg(x: npt.NDArray[np.float64]) -> object: return -x[1] + @compiled def nd_len(x: npt.NDArray[np.float64]) -> int: return len(x) + @compiled def nd_set(x: npt.NDArray[np.float64], i: int, v: float) -> None: x[i] = v + @compiled def nd_copy(dst: npt.NDArray[np.float64], src: npt.NDArray[np.float64]) -> None: for i in range(len(dst)): dst[i] = src[i] + @compiled def nd_iset(x: npt.NDArray[np.int64], i: int, v: int) -> None: x[i] = v + @compiled def nd_get(x: npt.NDArray[np.float64], i: int) -> object: return x[i] + @compiled def nd_overflow(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def nd_invalid(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def nd_iover(x: npt.NDArray[np.int64]) -> object: return x[0] + x[1] + @compiled def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> float: @@ -135,6 +156,7 @@ def nd_energy(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDA return e + @compiled def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], m: npt.NDArray[np.float64]) -> None: @@ -145,20 +167,24 @@ def nd_copy7(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDAr m[i] = x[i] + @compiled def nd_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def nd_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def nd_param(x: npt.NDArray[np.float64], k: float, use: bool) -> object: if use: k = x[0] return k + @compiled def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: if use: v = x[0] @@ -167,36 +193,44 @@ def nd_branch(x: npt.NDArray[np.float64], use: bool) -> object: return v + @compiled def nd_tuple(x: npt.NDArray[np.int64]) -> tuple[int, int, bool]: return (x[0], 3, x[1] > 0) + @compiled def nd_abs(x: npt.NDArray[np.float64]) -> object: return abs(x[1]) + @compiled def nd_to_list(x: npt.NDArray[np.float64], dst: list[float]) -> None: dst[0] = x[0] + @compiled def nd_mixed_cmp(x: npt.NDArray[np.int64], f: float) -> object: if x[0] < f: return 1 return 0 + @compiled def whole_sum(x: npt.NDArray[np.float64]) -> object: return np.sum(x) + @compiled def whole_attr(x: npt.NDArray[np.float64]) -> object: return x.size + @compiled def whole_pass(x: npt.NDArray[np.float64]) -> object: return np.add(x, 1.0).sum() + @compiled def nd_int_impure(x: npt.NDArray[np.int64]) -> object: s = 0 for i in range(len(x)): @@ -205,83 +239,103 @@ def nd_int_impure(x: npt.NDArray[np.int64]) -> object: return s + @compiled def nd_scale(x: npt.NDArray[np.float64], k: float) -> None: for i in range(len(x)): x[i] = x[i] * k + @compiled def p_div(x: npt.NDArray[np.float64]) -> object: return x[0] / x[1] + @compiled def p_fdiv(x: npt.NDArray[np.float64]) -> object: return x[0] // x[1] + @compiled def p_mod(x: npt.NDArray[np.float64]) -> object: return x[0] % x[1] + @compiled def p_add(x: npt.NDArray[np.float64]) -> object: return x[0] + x[1] + @compiled def p_sub(x: npt.NDArray[np.float64]) -> object: return x[0] - x[1] + @compiled def p_mul(x: npt.NDArray[np.float64]) -> object: return x[0] * x[1] + @compiled def p_pdiv(x: npt.NDArray[np.float64], k: float) -> object: return k / x[0] + @compiled def p_idiv(x: npt.NDArray[np.int64]) -> object: return x[0] / x[1] + @compiled def p_ifdiv(x: npt.NDArray[np.int64]) -> object: return x[0] // x[1] + @compiled def p_imod(x: npt.NDArray[np.int64]) -> object: return x[0] % x[1] + @compiled def p_imixed(x: npt.NDArray[np.int64]) -> object: return x[0] // 3 + x[1] % 4 + @compiled def i_div(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] / x[1] + @compiled def i_fdiv(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] // x[1] + @compiled def i_mod(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] % x[1] + @compiled def i_add(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] + x[1] + @compiled def i_sub(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] - x[1] + @compiled def i_mul(x: npt.NDArray[np.float64]) -> None: x[2] = x[0] * x[1] + @compiled def i_plain(x: npt.NDArray[np.float64], k: float) -> None: x[2] = k / x[1] x[3] = k * 2.0 + @compiled def i_multiline(x: npt.NDArray[np.float64]) -> None: x[2] = (x[0] * x[1]) @@ -290,12 +344,14 @@ def i_multiline(x: npt.NDArray[np.float64]) -> None: + x[0] * x[1]) + @compiled def i_aug(x: npt.NDArray[np.float64]) -> None: x[0] *= x[1] x[1] /= x[2] x[2] += x[3] + @compiled def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: s = 0.0 for i in range(len(x)): @@ -304,21 +360,25 @@ def i_loop(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64]) -> float: return s + @compiled def i_mixed(x: npt.NDArray[np.float64], n: int) -> None: x[0] = n / x[1] x[2] = x[3] * n + @compiled def i_ret(x: npt.NDArray[np.float64]) -> object: x[2] = x[0] * x[1] return x[0] / x[1] + @compiled def i_ifdiv(x: npt.NDArray[np.int64]) -> object: x[2] = x[0] // x[1] return x[2] + @compiled def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -343,6 +403,7 @@ def nd_advance(dt: float, steps: int, x: npt.NDArray[np.float64], y: npt.NDArray z[i] += dt * vz[i] + @compiled def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], z: npt.NDArray[np.float64], vx: npt.NDArray[np.float64], vy: npt.NDArray[np.float64], vz: npt.NDArray[np.float64], @@ -357,10 +418,12 @@ def nd_energy_full(x: npt.NDArray[np.float64], y: npt.NDArray[np.float64], e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) return e # #261: np.bool_ arrays (1-byte items, format "?") + @compiled def nd_b_get(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + @compiled def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: for i in range(len(x)): if x[i]: @@ -368,6 +431,7 @@ def nd_b_any(x: npt.NDArray[np.bool_]) -> bool: return False + @compiled def nd_b_count(x: npt.NDArray[np.bool_]) -> int: c = 0 for i in range(len(x)): @@ -376,45 +440,55 @@ def nd_b_count(x: npt.NDArray[np.bool_]) -> int: return c + @compiled def nd_b_set(x: npt.NDArray[np.bool_], i: int, v: bool) -> None: x[i] = v + @compiled def nd_b_flip(x: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): x[i] = not x[i] + @compiled def nd_b_copy(a: npt.NDArray[np.bool_], b: npt.NDArray[np.bool_]) -> None: for i in range(len(a)): b[i] = a[i] + @compiled def nd_b_eq(x: npt.NDArray[np.bool_], y: npt.NDArray[np.bool_], i: int) -> object: return x[i] == y[i] + @compiled def nd_b_gt(x: npt.NDArray[np.float64], m: npt.NDArray[np.bool_]) -> None: for i in range(len(x)): m[i] = x[i] > 2.0 + @compiled def nd_b_len(x: npt.NDArray[np.bool_]) -> int: return len(x) + @compiled def nd_b_store_int(x: npt.NDArray[np.bool_], i: int, v: int) -> None: x[i] = v + @compiled def nd_b_store_float(x: npt.NDArray[np.bool_], i: int, v: float) -> None: x[i] = v + @compiled def nd_b_arith(x: npt.NDArray[np.bool_], i: int) -> object: return x[i] + 1 + @compiled def nd_b_whole(x: npt.NDArray[np.bool_]) -> object: return np.sum(x) ''') diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index d6693dcad..cd08794e3 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -40,6 +40,7 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, cost, frontend, pipeline, verify HERE = Path(__file__).resolve().parent @@ -51,10 +52,10 @@ BINARY_TREES = REPO / "benchmarks" / "typedpython" / "py" / "binary_trees.py" SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython class-method test module.""" + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -135,21 +136,25 @@ def no_annotation(self, x) -> int: return 1 + @compiled class Plain: def check(self) -> int: return 5 + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check_of(n: Node) -> int: return n.check() + @compiled def leaf_of(n: Node) -> bool: return n.leaf() ''') @@ -231,6 +236,7 @@ def test_the_methods_of_a_class_that_is_not_fixed_layout_are_not_considered(tmp_ def test_a_method_named_like_a_module_function_does_not_collide(tmp_path): m = lower(tmp_path, SOURCE + textwrap.dedent('''\ + @compiled def height(n: Node) -> int: return n.height() ''')) @@ -240,7 +246,7 @@ def height(n: Node) -> int: def test_a_method_with_a_foreign_self_annotation_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -258,7 +264,7 @@ def get(self: int) -> int: def test_a_method_defined_twice_in_the_class_body_stays_interpreted(tmp_path): src = textwrap.dedent('''\ - # typedpython: compiled + @compiled class A: __slots__ = ("x",) x: int @@ -277,7 +283,7 @@ def get(self) -> int: assert "more than once" in m.skipped["A.get"] -def test_an_at_compiled_class_compiles_its_methods_without_the_module_marker(tmp_path): +def test_an_at_compiled_class_compiles_its_methods(tmp_path): src = textwrap.dedent('''\ @compiled class A: @@ -887,7 +893,7 @@ def run_main(path_or_module, n): def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): - text = "# typedpython: compiled\n" + method_form(BINARY_TREES.read_text()) + text = decorate_compiled(method_form(BINARY_TREES.read_text())) path = tmp_path / "bt_method.py" path.write_text(text) result = pipeline.compile_module(path, tmp_path / "out") @@ -921,8 +927,8 @@ def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): @pytest.mark.parametrize("name", sorted(EXPECTED)) def test_the_benchmarks_compile_the_same_functions_as_before(tmp_path, name): path = tmp_path / f"{name}.py" - path.write_text("# typedpython: compiled\n" + (REPO / "benchmarks" / "typedpython" / "py" - / f"{name}.py").read_text()) + path.write_text(decorate_compiled((REPO / "benchmarks" / "typedpython" / "py" + / f"{name}.py").read_text())) proved, diags = verify.verify(frontend.lower(path)) proved, diags = cost.apply(proved, diags) assert {f.name for f in proved.functions} == EXPECTED[name] diff --git a/python-multiplatform-ksp/src/test/python/test_cost_rule.py b/python-multiplatform-ksp/src/test/python/test_cost_rule.py index db6c4999a..c540bb260 100644 --- a/python-multiplatform-ksp/src/test/python/test_cost_rule.py +++ b/python-multiplatform-ksp/src/test/python/test_cost_rule.py @@ -12,20 +12,19 @@ import pytest +from conftest import decorate_compiled from typedpython import cost, pipeline -HEADER = "# typedpython: compiled\n" - -OBJECT_ONLY = HEADER + textwrap.dedent(''' +OBJECT_ONLY = decorate_compiled(textwrap.dedent(''' def copy_words(src: list[str], dst: list[str]) -> str: w: str = "" for x in src: w = x + w dst.append(w) return w -''') +''')) -INT_LOOP = HEADER + textwrap.dedent(''' +INT_LOOP = decorate_compiled(textwrap.dedent(''' def word_of(word_id: int) -> str: x: int = word_id + 17576 w: str = "" @@ -33,7 +32,7 @@ def word_of(word_id: int) -> str: w = chr(97 + x % 26) + w x //= 26 return w -''') +''')) def build(tmp_path, source, **kw): @@ -67,7 +66,7 @@ def test_the_opt_out_compiles_the_same_function(tmp_path): assert names(r) == {"copy_words"} and r.extension is not None -MIXED = HEADER + textwrap.dedent(''' +MIXED = decorate_compiled(textwrap.dedent(''' def mixed(n: int, xs: list[str]) -> float: total: float = 0.0 for i in range(n): @@ -106,7 +105,7 @@ def native_loop_inside_object_loop(rows: list[list[str]]) -> float: for k in range(3): total += 0.5 * 2.0 return total -''') +''')) def test_one_native_loop_with_object_operations_compiles(tmp_path): @@ -148,10 +147,10 @@ def test_the_rule_is_what_keeps_it_interpreted(tmp_path, monkeypatch): test_an_object_only_loop_stays_interpreted_with_the_reason(tmp_path) -STRAIGHT_LINE = HEADER + textwrap.dedent(''' +STRAIGHT_LINE = decorate_compiled(textwrap.dedent(''' def label(n: int, name: str) -> str: return f"{name}:{n + 1}" -''') +''')) def test_a_straight_line_impure_function_still_compiles(tmp_path): diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index 51f1bca43..0d6bb643b 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -23,9 +23,9 @@ posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def spin(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -33,12 +33,14 @@ def spin(n: int) -> float: return acc +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -165,9 +167,9 @@ def handler(signum, frame): # ---- chunked polling for range loops (#206, SPEC N-8) --------------------------------------------- CHUNK_SOURCE = """\ -# typedpython: compiled +@compiled def brk(n: int, k: int) -> int: s: int = 0 last: int = -1 @@ -179,6 +181,7 @@ def brk(n: int, k: int) -> int: return s * 3 + last +@compiled def cont(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -188,6 +191,7 @@ def cont(n: int, m: int) -> int: return s +@compiled def ret(n: int, k: int) -> int: for i in range(n): if i == k: @@ -195,6 +199,7 @@ def ret(n: int, k: int) -> int: return -1 +@compiled def down(n: int) -> int: s: int = 0 for i in range(n, 0, -3): @@ -202,6 +207,7 @@ def down(n: int) -> int: return s +@compiled def down1(n: int) -> int: s: int = 0 for i in range(n, -1, -1): @@ -209,6 +215,7 @@ def down1(n: int) -> int: return s +@compiled def up3(a: int, b: int) -> int: s: int = 0 for i in range(a, b, 3): @@ -216,6 +223,7 @@ def up3(a: int, b: int) -> int: return s +@compiled def nested(n: int, m: int) -> int: s: int = 0 for i in range(n): @@ -226,6 +234,7 @@ def nested(n: int, m: int) -> int: return s +@compiled def brk_in_while(n: int) -> int: s: int = 0 for i in range(n): @@ -238,6 +247,7 @@ def brk_in_while(n: int) -> int: return s +@compiled def huge(a: int, b: int, cap: int) -> int: c: int = 0 for i in range(a, b): @@ -247,6 +257,7 @@ def huge(a: int, b: int, cap: int) -> int: return c +@compiled def huge_last(a: int, b: int, cap: int) -> int: c: int = 0 last: int = 0 @@ -346,7 +357,7 @@ def test_range_loop_polls_once_per_chunk(chunk_module): def test_while_loop_keeps_its_per_iteration_countdown(tmp_path): path = tmp_path / "w.py" - path.write_text("# typedpython: compiled\n\n\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" + path.write_text("\n\n@compiled\ndef w(n: int) -> int:\n i: int = 0\n while i < n:\n" " i += 1\n return i\n") body = _function_c(_generated_c(path), "w") lines = body.splitlines() @@ -379,9 +390,9 @@ def test_keyboard_interrupt_reaches_a_long_range_loop_in_chunks(extension): MID_SOURCE = """\ -# typedpython: compiled +@compiled def mid(n: int) -> float: x: float = 1.0 for k in range(n): diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index f1e43a1ac..d448e3b41 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -25,45 +25,53 @@ posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") SOURCE = """\ -# typedpython: compiled +@compiled def helper(x: int) -> float: d: dict[str, int] = {} d['a'] = x return d['a'] * 2.5 +@compiled def leaf(i: int) -> float: return 1.0 / (i + 1) +@compiled def mid(i: int) -> float: return leaf(i) + leaf(i + 1) +@compiled def top(i: int) -> float: return mid(i) * 2.0 +@compiled def objy(i: int) -> float: return float(helper(i)) + leaf(i) +@compiled def viaobj(i: int) -> float: return objy(i) + 1.0 +@compiled def rec(n: int) -> int: if n == 0: return 0 return 1 + rec(n - 1) +@compiled def callrec(n: int) -> int: return rec(n) + 1 +@compiled def loopy(n: int) -> float: s: float = 0.0 for i in range(n): @@ -71,14 +79,17 @@ def loopy(n: int) -> float: return s +@compiled def outer(n: int) -> float: return loopy(n) + 1.0 +@compiled def cube(x: int) -> float: return float(x * x * x) + leaf(x) +@compiled def fill(x: int, out: list[float]) -> None: out[0] = cube(x) """ diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index 378395e78..e954c03ba 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -33,9 +33,9 @@ FREE_THREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) SOURCE = '''\ -# typedpython: compiled +@compiled class Cell: __slots__ = ("a", "b") a: object @@ -46,6 +46,7 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled class Twin: __slots__ = ("a", "b") a: object @@ -56,18 +57,22 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def geta(c: Cell) -> object: return c.a +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def mk(a: object, b: object) -> Cell: return Cell(a, b) +@compiled def fresh(a: object, n: int) -> object: c = Cell(a, None) keep: object = c # read as a value: c is a real object, not virtual (N-12a) @@ -77,11 +82,13 @@ def fresh(a: object, n: int) -> object: return s +@compiled def vpick(a: object, b: object) -> object: c = Cell(a, b) return c.b +@compiled def total(xs: list[float], rounds: int) -> float: s: float = 0.0 for r in range(rounds): @@ -90,24 +97,29 @@ def total(xs: list[float], rounds: int) -> float: return s +@compiled def fill(xs: list[float], v: float) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge(xs: list[float]) -> float: return xs[0] - xs[len(xs) - 1] +@compiled def fill_int(xs: list[int], v: int) -> None: for i in range(len(xs)): xs[i] = v +@compiled def edge_int(xs: list[int]) -> int: return xs[0] - xs[len(xs) - 1] +@compiled def rotate(xs: list[float]) -> None: if len(xs) > 1: first: float = xs[0] @@ -116,6 +128,7 @@ def rotate(xs: list[float]) -> None: xs[len(xs) - 1] = first +@compiled def zero2(xs: list[int], ys: list[int]) -> int: for i in range(len(xs)): xs[i] = 0 @@ -124,10 +137,12 @@ def zero2(xs: list[int], ys: list[int]) -> int: return 0 +@compiled def pair_edge(xs: list[int], ys: list[int]) -> int: return xs[0] - ys[len(ys) - 1] +@compiled def three(a: list[int], b: list[int], c: list[int]) -> int: return len(a) + len(b) + len(c) @@ -135,6 +150,7 @@ def three(a: list[int], b: list[int], c: list[int]) -> int: K: int = 3 +@compiled def usek(x: int) -> int: return x + K ''' @@ -244,7 +260,7 @@ def test_three_list_parameters_stay_interpreted(built): def test_front_end_gives_a_clear_reason(tmp_path): p = tmp_path / "m.py" - p.write_text("# typedpython: compiled\n\n\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" + p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" " return len(a)\n") m = frontend.lower(p) assert not m.functions diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 8b2a55fd9..fe8f4d94c 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -12,6 +12,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, @@ -54,7 +55,7 @@ def nodes(root, cls): def test_scalar_loop_lowers_to_for_range(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def total(n: int, x: float) -> float: acc: float = 0.0 for k in range(n): @@ -633,16 +634,18 @@ def f(x: float) -> object: assert Global(OBJ, "K") in nodes(f, Global) -def test_module_marker_compiles_every_module_level_def(tmp_path): +def test_compiled_marks_every_decorated_module_level_def(tmp_path): m = lower(tmp_path, """ - # typedpython: compiled + @compiled def a(x: float) -> float: return x + @compiled def b(x: float) -> float: return -x + @compiled @staticmethod def c(x: float) -> float: return x @@ -651,7 +654,19 @@ def c(x: float) -> float: assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] -def test_without_marker_only_compiled_functions_are_lowered(tmp_path): +def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): + m = lower(tmp_path, """ + # typedpython: compiled + def a(x: float) -> float: + return x + + def b(x: float) -> float: + return -x + """) + assert m.functions == () and m.skipped == {} + + +def test_without_a_decorator_only_compiled_functions_are_lowered(tmp_path): m = lower(tmp_path, """ x: int = 1 # typedpython: compiled @@ -828,7 +843,7 @@ def f(n: int) -> int: def test_nbody_kernels_lower(tmp_path): path = tmp_path / "nbody.py" shutil.copy(BENCH / "nbody.py", path) - path.write_text("# typedpython: compiled\n" + path.read_text()) + path.write_text(decorate_compiled(path.read_text())) m = frontend.lower(path) for name in ("advance", "energy", "offset_momentum"): function(m, name) @@ -847,7 +862,7 @@ def test_nbody_kernels_lower(tmp_path): @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_spectral_norm_kernels_lower(tmp_path): path = tmp_path / "spectral_norm.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "spectral_norm.py").read_text()) + path.write_text(decorate_compiled((BENCH / "spectral_norm.py").read_text())) m = frontend.lower(path) eval_a = function(m, "eval_a") assert eval_a.pure and eval_a.may_deopt @@ -1048,7 +1063,7 @@ def f(a: int, b: int) -> {annotation}: @pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") def test_fannkuch_lowers(tmp_path): path = tmp_path / "fannkuch.py" - path.write_text("# typedpython: compiled\n" + (BENCH / "fannkuch.py").read_text()) + path.write_text(decorate_compiled((BENCH / "fannkuch.py").read_text())) m = frontend.lower(path) assert "fannkuch" in [f.name for f in m.functions], m.skipped.get("fannkuch") f = function(m, "fannkuch") diff --git a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py index 33c0473cc..a2daaeed9 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py @@ -18,6 +18,7 @@ import pytest +from conftest import decorate_compiled from typedpython import frontend from typedpython.ir import ( And, Assign, Box, Call, CallObject, CheckExact, ClassDecl, Const, ExprStmt, FieldGet, FieldSet, @@ -28,8 +29,6 @@ BINARY_TREES = Path(__file__).resolve().parents[4] / "benchmarks/typedpython/py/binary_trees.py" SLOTS_LINE = ' __slots__ = ("left", "right")\n' -MARK = "# typedpython: compiled\n" - POINT = """ class P: __slots__ = ("x", "y") @@ -44,7 +43,7 @@ def __init__(self, x: float, y: float) -> None: def lower(tmp_path, source, name="mod.py"): path = tmp_path / name - path.write_text(MARK + textwrap.dedent(source).lstrip("\n")) + path.write_text(decorate_compiled(textwrap.dedent(source).lstrip("\n"))) return frontend.lower(path) @@ -573,7 +572,7 @@ def test_binary_trees_with_slots_lowers_make_and_check(tmp_path): source = BINARY_TREES.read_text() assert "class Node:\n" + SLOTS_LINE in source # the benchmark itself declares the slots path = tmp_path / "binary_trees.py" - path.write_text(MARK + source) + path.write_text(decorate_compiled(source)) m = frontend.lower(path) assert cls(m, "Node") == ClassDecl("Node", ("left", "right"), True) @@ -598,7 +597,7 @@ def test_binary_trees_without_slots_stays_object_code(tmp_path): path = tmp_path / "binary_trees.py" source = BINARY_TREES.read_text() assert SLOTS_LINE in source - path.write_text(MARK + source.replace(SLOTS_LINE, "", 1)) # the run.py no-slots baseline + path.write_text(decorate_compiled(source.replace(SLOTS_LINE, "", 1))) # the run.py no-slots baseline m = frontend.lower(path) assert m.classes == () assert "`__slots__`" in m.skipped["Node"] diff --git a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py index 045a593e3..64260ddf7 100644 --- a/python-multiplatform-ksp/src/test/python/test_interval_proofs.py +++ b/python-multiplatform-ksp/src/test/python/test_interval_proofs.py @@ -330,39 +330,44 @@ def test_a_mod_result_can_feed_a_proof(): # --- 2. end to end --------------------------------------------------------------------------------- SOURCE = '''\ -# typedpython: compiled +@compiled def dec_if_positive(n: int) -> int: if n > 0: return n - 1 return -1 +@compiled def dec_after_guard(n: int) -> int: if n <= 0: return 0 return n - 1 +@compiled def inc_below_max(n: int) -> int: if n < 9223372036854775807: return n + 1 return 0 +@compiled def inc_below_100(n: int) -> int: if n < 100: return n + 1 return 0 +@compiled def inc_if_positive(n: int) -> int: if n > 0: return n + 1 return 0 +@compiled def reverse_gap(a: int, b: int) -> int: lo: int = a hi: int = b @@ -374,6 +379,7 @@ def reverse_gap(a: int, b: int) -> int: return hi - lo + steps +@compiled def killed_guard(n: int, a: list[int]) -> int: if n < 100: n = a[0] @@ -381,6 +387,7 @@ def killed_guard(n: int, a: list[int]) -> int: return 0 +@compiled def killed_in_loop(c: int, a: list[int]) -> int: n: int = a[0] t: int = 0 @@ -391,6 +398,7 @@ def killed_in_loop(c: int, a: list[int]) -> int: return t +@compiled def parity(n: int) -> int: k: int = 0 s: int = 0 @@ -401,11 +409,13 @@ def parity(n: int) -> int: return s +@compiled def const_local() -> int: x: int = 41 return x + 1 +@compiled def range_shifts(a: int, b: int) -> int: if a < 0: return 0 @@ -415,26 +425,31 @@ def range_shifts(a: int, b: int) -> int: return t +@compiled def guarded_shift(n: int) -> int: if n >= 0: return n // 16 + n % 16 return 0 +@compiled def negative_shift(n: int) -> int: return n // 4 + n % 8 +@compiled def odd_divisor(n: int) -> int: if n >= 0: return n // 3 + n % 6 return 0 +@compiled def mod_then_add(n: int) -> int: return n % 10 + 1 +@compiled def pair_sums(a: list[int]) -> int: best: int = 0 for i in range(len(a)): diff --git a/python-multiplatform-ksp/src/test/python/test_module_report.py b/python-multiplatform-ksp/src/test/python/test_module_report.py index 8625cf18d..86ead4b97 100644 --- a/python-multiplatform-ksp/src/test/python/test_module_report.py +++ b/python-multiplatform-ksp/src/test/python/test_module_report.py @@ -8,16 +8,17 @@ import textwrap from pathlib import Path +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, pipeline, verify -SRC = "# typedpython: compiled\n" + textwrap.dedent("""\ +SRC = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 def boom(s: str) -> str: raise ValueError("boom " + s) - """) + """)) def make_project(root: Path) -> Path: @@ -95,14 +96,14 @@ def test_a_moved_file_with_the_same_content_does_not_reuse_the_extension(tmp_pat # --- #128 ---------------------------------------------------------------------------------------- -MIXED = "# typedpython: compiled\n" + textwrap.dedent("""\ +MIXED = decorate_compiled(textwrap.dedent("""\ def ok(n: int) -> int: return n + 1 def greet(name: str) -> str: return name[1:] - """) + """)) def build_one(tmp_path, rel, source, cache="cache"): @@ -119,20 +120,20 @@ def test_kind_compiled(tmp_path): assert m.kind == "compiled" and m.extension is not None [item] = m.skipped_items assert item.name == "greet" and item.file == "pkg/mixed.py" - assert item.line == 7 and not item.message.startswith("line ") - assert m.skipped["greet"].startswith("line 7: ") # ModuleResult.skipped unchanged + assert item.line == 8 and not item.message.startswith("line ") + assert m.skipped["greet"].startswith("line 8: ") # ModuleResult.skipped unchanged def test_kind_nothing_compiled_with_structured_items(tmp_path): _, _, m = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") assert m.kind == "nothing_compiled" and m.extension is None [item] = m.skipped_items - assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 4) + assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 3) assert item.message and not re.match(r"line \d+", item.message) -def test_kind_not_marked_when_there_is_neither_marker_nor_decorator(tmp_path): +def test_kind_not_marked_when_there_is_no_decorator(tmp_path): _, _, m = build_one(tmp_path, "pkg/unmarked.py", "def f(x: int) -> int:\n return x\n") assert m.kind == "not_marked" and m.extension is None and m.skipped_items == [] @@ -145,15 +146,15 @@ def test_a_compiled_decorator_alone_opts_in(tmp_path): def test_class_items_carry_name_and_line(tmp_path): - src = ("# typedpython: compiled\n\n\nclass Node:\n __slots__ = ('a',)\n" + src = ("@compiled\nclass Node:\n __slots__ = ('a',)\n" " def __init__(self, a: int) -> None:\n self.a = a\n") _, _, m = build_one(tmp_path, "pkg/cls.py", src) cls = [i for i in m.skipped_items if i.name == "Node"] - assert cls and cls[0].line == 4 and cls[0].file == "pkg/cls.py" + assert cls and cls[0].line == 2 and cls[0].file == "pkg/cls.py" def test_verifier_skips_carry_a_line_and_no_prefix(tmp_path): - src = "# typedpython: compiled\n\n\ndef f(x: int) -> int:\n return g(x)\n\n\ndef g(x: int) -> int:\n return x\n" + src = "@compiled\ndef f(x: int) -> int:\n return g(x)\n\n\n@compiled\ndef g(x: int) -> int:\n return x\n" _, _, m = build_one(tmp_path, "pkg/v.py", src) for i in m.skipped_items: assert isinstance(i.line, int) and not i.message.startswith("line ") @@ -170,7 +171,7 @@ def test_structure_round_trips_through_the_cache_for_reused_modules(tmp_path): def test_not_marked_and_nothing_compiled_survive_reuse(tmp_path): root, path, _ = build_one(tmp_path, "pkg/plain.py", - "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s[1:]\n") + "@compiled\ndef f(s: str) -> str:\n return s[1:]\n") again = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] assert again.status == "reused" and again.kind == "nothing_compiled" and again.skipped_items root2, path2, _ = build_one(tmp_path, "pkg/un.py", "def f(x: int) -> int:\n return x\n") @@ -191,5 +192,5 @@ def test_cli_build_prints_file_line_name_message_and_not_marked(tmp_path): [sys.executable, "-m", "typedpython", "build", str(root), "--cache", str(tmp_path / "c")], capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(src))) assert out.returncode == 0, out.stdout + out.stderr - assert re.search(r"^\s*pkg/mixed\.py:7: greet: \S", out.stdout, re.M), out.stdout + assert re.search(r"^\s*pkg/mixed\.py:8: greet: \S", out.stdout, re.M), out.stdout assert re.search(r"pkg/un\.py.*not marked for compilation", out.stdout), out.stdout diff --git a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py index f2ceeb875..2da246f90 100644 --- a/python-multiplatform-ksp/src/test/python/test_native_int_impure.py +++ b/python-multiplatform-ksp/src/test/python/test_native_int_impure.py @@ -28,13 +28,13 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, cost, frontend, ir, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled import math as GMOD @@ -42,14 +42,17 @@ MOD: int = 2147483647 + @compiled def _one() -> int: return 1 + @compiled def _two() -> int: return 2 + @compiled class GTYPE: pass @@ -57,6 +60,7 @@ class GTYPE: GFN = _one + @compiled def add2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + b @@ -66,6 +70,7 @@ def add2(log: list[object], a: int, b: int) -> object: return r + @compiled def sub2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - b @@ -75,6 +80,7 @@ def sub2(log: list[object], a: int, b: int) -> object: return r + @compiled def mul2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a * b @@ -84,6 +90,7 @@ def mul2(log: list[object], a: int, b: int) -> object: return r + @compiled def neg2(log: list[object], a: int) -> object: log.append("start") r: int = -a @@ -93,6 +100,7 @@ def neg2(log: list[object], a: int) -> object: return r + @compiled def floordiv2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -104,6 +112,7 @@ def floordiv2(log: list[object], a: int, b: int) -> object: return r + @compiled def mod2(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a - 1 @@ -115,6 +124,7 @@ def mod2(log: list[object], a: int, b: int) -> object: return r + @compiled def lsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a << b @@ -122,6 +132,7 @@ def lsh(log: list[object], a: int, b: int) -> object: return r + @compiled def rsh(log: list[object], a: int, b: int) -> object: log.append("start") r: int = a + 1 @@ -131,6 +142,7 @@ def rsh(log: list[object], a: int, b: int) -> object: return r + @compiled def roundtrip(log: list[object], a: int) -> object: log.append(a) x: int = a @@ -145,6 +157,7 @@ def roundtrip(log: list[object], a: int) -> object: return x + @compiled def compare(log: list[object], a: int, b: int) -> None: log.append("start") x: int = a + 1 @@ -167,6 +180,7 @@ def compare(log: list[object], a: int, b: int) -> None: log.append("y-zero") + @compiled def fact(log: list[object], n: int) -> object: log.append(n) f: int = 1 @@ -176,6 +190,7 @@ def fact(log: list[object], n: int) -> object: return f + @compiled def grow(log: list[object], a: int, k: int) -> object: x: int = a i: int = 0 @@ -186,16 +201,19 @@ def grow(log: list[object], a: int, k: int) -> object: return x + @compiled def consts(log: list[object]) -> None: log.append(7000000000) log.append(5) log.append(-7000000000) + @compiled def next_state(s: int) -> int: return (s * 48271) % MOD + @compiled def drive(log: list[object], seed: int, n: int) -> object: s: int = seed + 1 for _ in range(n): @@ -204,37 +222,44 @@ def drive(log: list[object], seed: int, n: int) -> object: return s + @compiled def read_limit(log: list[object]) -> object: log.append(1) return LIMIT + @compiled def read_twice(log: list[object], cb: object) -> None: log.append(LIMIT) cb() log.append(LIMIT) + @compiled def use_builtin(log: list[object], s: str) -> object: log.append(1) return len(s) + @compiled def read_fn(log: list[object]) -> object: log.append(1) return GFN + @compiled def read_type(log: list[object]) -> object: log.append(1) return GTYPE + @compiled def read_mod(log: list[object]) -> object: log.append(1) return GMOD + @compiled def call_g(x: object) -> object: return GFN() ''') @@ -314,7 +339,7 @@ def test_int_locals_of_impure_functions_are_native(built): def test_an_int_fed_from_an_object_stays_an_object(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def f(log: list[object], g: object) -> object: log.append(1) n: int = g() @@ -656,7 +681,7 @@ def test_a_cache_that_is_never_invalidated_is_caught(tmp_path): def test_an_int_loop_is_native_work_for_the_cost_rule(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def digits(log: list[object], n: int) -> None: x: int = n + 1 while x > 0: @@ -672,7 +697,7 @@ def digits(log: list[object], n: int) -> None: def test_an_object_only_loop_is_still_rejected(tmp_path): src = textwrap.dedent(''' - # typedpython: compiled + @compiled def words(xs: list[object], ys: list[object]) -> None: for x in xs: ys.append(x) @@ -688,7 +713,7 @@ def words(xs: list[object], ys: list[object]) -> None: @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) path = d / "wordfreq.py" path.write_text(src) proved, diags = verify.verify(frontend.lower(path)) diff --git a/python-multiplatform-ksp/src/test/python/test_osr_exit.py b/python-multiplatform-ksp/src/test/python/test_osr_exit.py index fbfbdd3cf..227d50ecd 100644 --- a/python-multiplatform-ksp/src/test/python/test_osr_exit.py +++ b/python-multiplatform-ksp/src/test/python/test_osr_exit.py @@ -17,11 +17,11 @@ from typedpython import cbuild, cgen, frontend, verify SOURCE = textwrap.dedent(''' - # typedpython: compiled SCALE = 2.0 + @compiled def scale_all(x: list[float]) -> float: s = 0.0 for i in range(len(x)): @@ -30,6 +30,7 @@ def scale_all(x: list[float]) -> float: return s + @compiled def nested(x: list[float], n: int) -> int: hits = 0 for r in range(n): @@ -43,6 +44,7 @@ def nested(x: list[float], n: int) -> int: return hits + @compiled def countdown(x: list[int]) -> int: k = 0 total = 0 diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index 55fc184c6..e86605b69 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -33,26 +33,29 @@ PAR_RULE = "verify/par-loop" SOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b +@compiled def scale_unmarked(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b # typedpython: parallel +@compiled def ramp(xs: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = -(float(i) * a) + xs[i] # typedpython: parallel +@compiled def two_stores(xs: list[float], ys: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = xs[i] + a @@ -60,12 +63,14 @@ def two_stores(xs: list[float], ys: list[float], a: float) -> None: # typedpython: parallel +@compiled def neighbour(xs: list[float]) -> None: for i in range(len(xs) - 1): xs[i] = xs[i + 1] # typedpython: parallel +@compiled def total(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -73,23 +78,27 @@ def total(xs: list[float]) -> float: return s +@compiled def helper(x: float) -> float: return x * 2.0 # typedpython: parallel +@compiled def with_call(xs: list[float]) -> None: for i in range(len(xs)): xs[i] = helper(xs[i]) # typedpython: parallel +@compiled def divides(xs: list[float], d: float) -> None: for i in range(len(xs)): xs[i] = xs[i] / d # typedpython: parallel +@compiled def local_fill(a: float) -> float: t = [0.0] * 300000 for i in range(len(t)): @@ -98,6 +107,7 @@ def local_fill(a: float) -> float: # typedpython: parallel +@compiled def local_dyn(n: int, a: float) -> float: t = [1.0] * n for i in range(len(t)): @@ -438,9 +448,9 @@ def sh(): # ===================================================================================================== RSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def r_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -449,6 +459,7 @@ def r_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_sum_init(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -457,6 +468,7 @@ def r_sum_init(xs: list[int], s0: int) -> int: # typedpython: parallel +@compiled def r_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -465,6 +477,7 @@ def r_prod(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def r_sum_mul(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -473,6 +486,7 @@ def r_sum_mul(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def r_two(xs: list[int]) -> int: a: int = 0 b: int = 7 @@ -483,6 +497,7 @@ def r_two(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_range_sum(xs: list[int]) -> int: s: int = 5 for i in range(len(xs)): @@ -491,6 +506,7 @@ def r_range_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def r_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -501,6 +517,7 @@ def r_local(n: int) -> int: return s +@compiled def r_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -509,6 +526,7 @@ def r_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -517,6 +535,7 @@ def n_acc_read(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -526,6 +545,7 @@ def n_two_ops(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_float(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -534,6 +554,7 @@ def n_float(xs: list[float]) -> float: # typedpython: parallel +@compiled def n_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -544,6 +565,7 @@ def n_copy(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_mixed(xs: list[int]) -> int: t = [0.0] * len(xs) s: int = 0 @@ -554,6 +576,7 @@ def n_mixed(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -562,6 +585,7 @@ def n_div(xs: list[int], d: int) -> int: # typedpython: parallel +@compiled def n_shift_step(xs: list[int]) -> int: s: int = 0 for i in range(0, len(xs), 2): @@ -859,9 +883,9 @@ def test_no_reduction_region_without_a_marker(tmp_path): # ===================================================================================================== MSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def mn_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -871,6 +895,7 @@ def mn_i(xs: list[int]) -> int: # typedpython: parallel +@compiled def mx_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -880,6 +905,7 @@ def mx_i(xs: list[int]) -> int: # typedpython: parallel +@compiled def mn_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -889,6 +915,7 @@ def mn_f(xs: list[float]) -> float: # typedpython: parallel +@compiled def mx_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -898,6 +925,7 @@ def mx_f(xs: list[float]) -> float: # typedpython: parallel +@compiled def mn_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -907,6 +935,7 @@ def mn_f_flipped(xs: list[float], init: float) -> float: # typedpython: parallel +@compiled def mx_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -916,6 +945,7 @@ def mx_f_flipped(xs: list[float], init: float) -> float: # typedpython: parallel +@compiled def mn_i_expr(xs: list[int], c: int) -> int: m = c for i in range(len(xs)): @@ -925,6 +955,7 @@ def mn_i_expr(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def mn_f_expr(xs: list[float], a: float) -> float: m = a for i in range(len(xs)): @@ -934,6 +965,7 @@ def mn_f_expr(xs: list[float], a: float) -> float: # typedpython: parallel +@compiled def mn_and_sum(xs: list[int]) -> int: m = xs[0] s = 0 @@ -945,6 +977,7 @@ def mn_and_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def two_ext(xs: list[float]) -> float: lo = xs[0] hi = xs[0] @@ -957,6 +990,7 @@ def two_ext(xs: list[float]) -> float: # typedpython: parallel +@compiled def mn_i_init(xs: list[int], init: int) -> int: m = init for i in range(len(xs)): @@ -966,6 +1000,7 @@ def mn_i_init(xs: list[int], init: int) -> int: # typedpython: parallel +@compiled def mn_i_big_e(xs: list[int], c: int) -> int: m = 0 for i in range(len(xs)): @@ -974,6 +1009,7 @@ def mn_i_big_e(xs: list[int], c: int) -> int: return m +@compiled def mn_unmarked(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -983,6 +1019,7 @@ def mn_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_le(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -992,6 +1029,7 @@ def n_le(xs: list[float]) -> float: # typedpython: parallel +@compiled def n_other_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1001,6 +1039,7 @@ def n_other_value(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_else(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1013,6 +1052,7 @@ def n_else(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_two_in_then(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1024,6 +1064,7 @@ def n_two_in_then(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_acc_in_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1034,6 +1075,7 @@ def n_acc_in_value(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_cond_other(xs: list[int], t: int) -> int: m = xs[0] for i in range(len(xs)): @@ -1043,6 +1085,7 @@ def n_cond_other(xs: list[int], t: int) -> int: # typedpython: parallel +@compiled def n_mixed_types(xs: list[int], init: float) -> float: m = init for i in range(len(xs)): @@ -1052,6 +1095,7 @@ def n_mixed_types(xs: list[int], init: float) -> float: # typedpython: parallel +@compiled def n_acc_reads_acc(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1063,6 +1107,7 @@ def n_acc_reads_acc(xs: list[int]) -> int: # typedpython: parallel +@compiled def n_div(xs: list[float], d: float) -> float: m = xs[0] for i in range(len(xs)): @@ -1490,9 +1535,9 @@ def spin(): # ===================================================================================================== BSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def b_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1501,6 +1546,7 @@ def b_and(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1509,6 +1555,7 @@ def b_or(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1517,6 +1564,7 @@ def b_xor(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_and_swapped(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -1525,6 +1573,7 @@ def b_and_swapped(xs: list[int], s0: int) -> int: # typedpython: parallel +@compiled def b_or_swapped(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1533,6 +1582,7 @@ def b_or_swapped(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_xor_expr(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1541,6 +1591,7 @@ def b_xor_expr(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def b_or_not(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1549,6 +1600,7 @@ def b_or_not(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_mix(xs: list[int]) -> int: a: int = 0 x: int = 5 @@ -1564,6 +1616,7 @@ def b_mix(xs: list[int]) -> int: # typedpython: parallel +@compiled def b_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -1574,6 +1627,7 @@ def b_local(n: int) -> int: return s +@compiled def b_unmarked(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1582,6 +1636,7 @@ def b_unmarked(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1590,6 +1645,7 @@ def nb_acc_read(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1599,6 +1655,7 @@ def nb_two_ops(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_and_then_add(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1608,6 +1665,7 @@ def nb_and_then_add(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_shift(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1616,6 +1674,7 @@ def nb_shift(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_rshift(xs: list[int], k: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1624,6 +1683,7 @@ def nb_rshift(xs: list[int], k: int) -> int: # typedpython: parallel +@compiled def nb_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -1634,6 +1694,7 @@ def nb_copy(xs: list[int]) -> int: # typedpython: parallel +@compiled def nb_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1794,15 +1855,16 @@ def test_bitwise_equals_interpreter_for_every_thread_count(bbuilt, threads): # ===================================================================================================== TSOURCE = '''\ -# typedpython: compiled # typedpython: parallel +@compiled def t_scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b # typedpython: parallel +@compiled def t_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1811,6 +1873,7 @@ def t_sum(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -1819,6 +1882,7 @@ def t_prod(xs: list[int], c: int) -> int: # typedpython: parallel +@compiled def t_min(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1828,6 +1892,7 @@ def t_min(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_max(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1837,6 +1902,7 @@ def t_max(xs: list[float]) -> float: # typedpython: parallel +@compiled def t_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1845,6 +1911,7 @@ def t_and(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1853,6 +1920,7 @@ def t_or(xs: list[int]) -> int: # typedpython: parallel +@compiled def t_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 6efbeb14d..ed326eb4c 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -20,9 +20,9 @@ def interpreted(path: Path): SCALAR = """\ -# typedpython: compiled +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -30,6 +30,7 @@ def total(n: int) -> float: return acc +@compiled def fact(n: int) -> int: r: int = 1 for k in range(2, n + 1): @@ -53,6 +54,7 @@ def test_scalar_module_compiles_and_matches_cpython(write, tmp_path): def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): path = write("mixed.py", SCALAR + """ +@compiled def greet(name: str) -> str: return name[1:] """) @@ -68,16 +70,16 @@ def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path) def test_gate_errors_block_compilation(write, tmp_path): - path = write("bad.py", "# typedpython: compiled\n\ndef f(x):\n return x\n") + path = write("bad.py", "@compiled\ndef f(x):\n return x\n") with pytest.raises(pipeline.CompileError, match="implicit-any-parameter"): pipeline.compile_module(path, tmp_path / "out") DEMO = """\ -# typedpython: compiled import sys +@compiled def total(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -85,6 +87,7 @@ def total(n: int) -> float: return acc +@compiled def main() -> None: print(f"{total(int(sys.argv[1])):.3f}") """ @@ -103,15 +106,16 @@ def test_demo_reports_identical_output(write, tmp_path): ARRAY_CALLBACK = """\ -# typedpython: compiled from typing import Callable +@compiled def first_after(xs: list[float], cb: Callable[[], None]) -> float: cb() return xs[0] +@compiled def fill_then(xs: list[float], cb: Callable[[], None]) -> None: xs[1] = 5.0 cb() @@ -138,9 +142,9 @@ def cb2(): GC_REACHABLE = """\ -# typedpython: compiled +@compiled class Cell: a: object b: object @@ -151,10 +155,12 @@ def __init__(self, a: object, b: object) -> None: self.b = b +@compiled def seta(c: Cell, v: object) -> None: c.a = v +@compiled def fresh(a: object, n: int) -> object: c = Cell(None, None) seta(c, a) diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py index dd7cfad86..3735df9bc 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -9,25 +9,24 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, frontend, pipeline -HEADER = "# typedpython: compiled\n" - -B = HEADER + textwrap.dedent("""\ +B = decorate_compiled(textwrap.dedent("""\ def twice(n: int) -> int: return n * 2 - """) -A = HEADER + textwrap.dedent("""\ + """)) +A = decorate_compiled(textwrap.dedent("""\ import b def inc(n: int) -> int: return n + 1 - """) -C = HEADER + textwrap.dedent("""\ + """)) +C = decorate_compiled(textwrap.dedent("""\ def half(x: float) -> float: return x * 0.5 - """) + """)) def interpreted(path: Path): @@ -133,7 +132,7 @@ def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): root = tmp_path / "proj" root.mkdir() - (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s[1:]\n") + (root / "plain.py").write_text("@compiled\ndef f(s: str) -> str:\n return s[1:]\n") cache = tmp_path / "cache" first = pipeline.compile_project([root / "plain.py"], root, cache) m = first.modules[0] @@ -151,7 +150,7 @@ def test_interface_describes_the_verified_module_not_the_lowered_one(tmp_path, m root = tmp_path / "proj" root.mkdir() - (root / "dep.py").write_text(HEADER + "def ok(n: int) -> int:\n return n + 1\n") + (root / "dep.py").write_text("@compiled\ndef ok(n: int) -> int:\n return n + 1\n") real_lower = frontend.lower seen = {} diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index 289e1de66..49901b0e6 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -20,15 +20,16 @@ PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" SOURCE = """\ -# typedpython: compiled +@compiled def fib(n: int) -> int: if n < 2: return n return fib(n - 1) + fib(n - 2) +@compiled def ack(m: int, n: int) -> int: if m == 0: return n + 1 @@ -37,30 +38,35 @@ def ack(m: int, n: int) -> int: return ack(m - 1, ack(m, n - 1)) +@compiled def is_even(n: int) -> bool: if n == 0: return True return is_odd(n - 1) +@compiled def is_odd(n: int) -> bool: if n == 0: return False return is_even(n - 1) +@compiled def down(n: int) -> int: if n == 0: return 0 return 1 + down(n - 1) +@compiled def pow2(n: int) -> int: if n == 0: return 1 return 2 * pow2(n - 1) +@compiled def fact(n: int) -> int: if n < 2: return 1 diff --git a/python-multiplatform-ksp/src/test/python/test_regions.py b/python-multiplatform-ksp/src/test/python/test_regions.py index 0246fd207..ac7aad6c8 100644 --- a/python-multiplatform-ksp/src/test/python/test_regions.py +++ b/python-multiplatform-ksp/src/test/python/test_regions.py @@ -37,12 +37,12 @@ FAIL_AFTER = "__typedpython_region_fail_after__" # (n) -> the nth region object allocation fails; -1 off SOURCE = textwrap.dedent('''\ - # typedpython: compiled """typedpython region test module.""" KEPT = None + @compiled class Node: __slots__ = ("left", "right") left: "Node | None" @@ -53,12 +53,14 @@ def __init__(self, left: "Node | None", right: "Node | None") -> None: self.right = right + @compiled def make(depth: int) -> Node: if depth > 0: return Node(make(depth - 1), make(depth - 1)) return Node(None, None) + @compiled def check(node: Node) -> int: left: Node | None = node.left right: Node | None = node.right @@ -67,10 +69,12 @@ def check(node: Node) -> int: return 1 + check(left) + check(right) + @compiled def tree_sum(d: int) -> int: return check(make(d)) + @compiled def repeat(d: int, n: int) -> int: t: int = 0 for _ in range(n): @@ -78,35 +82,42 @@ def repeat(d: int, n: int) -> int: return t + @compiled def escapes(d: int) -> Node: return make(d) + @compiled def keeps(d: int) -> int: global KEPT KEPT = make(d) return check(KEPT) + @compiled def prints(d: int) -> int: n = make(d) print("unknown code while the tree is alive") return check(n) + @compiled def ident(d: int) -> int: return id(make(d)) * 0 + @compiled def same(d: int) -> bool: return make(d) is make(d) + @compiled def mixed(d: int) -> int: t = make(d) return check(make(d)) + check(t) + @compiled def noisy(d: int) -> object: print("unknown code") return check(make(d)) diff --git a/python-multiplatform-ksp/src/test/python/test_resume.py b/python-multiplatform-ksp/src/test/python/test_resume.py index 14115cc79..984c7b4b3 100644 --- a/python-multiplatform-ksp/src/test/python/test_resume.py +++ b/python-multiplatform-ksp/src/test/python/test_resume.py @@ -519,7 +519,7 @@ def lower(tmp_path, source): def test_frontend_gives_a_snapshot_holder_one_resume_per_loop(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def f(xs: list[float], n: int) -> float: t = 0.0 for i in range(n): @@ -572,7 +572,7 @@ def f(xs: list[float], n: int) -> float: def test_frontend_function_without_a_snapshot_has_no_resumes(tmp_path): module = lower(tmp_path, """ - # typedpython: compiled + @compiled def g(n: int) -> int: t = 0 for i in range(n): diff --git a/python-multiplatform-ksp/src/test/python/test_str_dict.py b/python-multiplatform-ksp/src/test/python/test_str_dict.py index 2362033c4..e6bc52c7b 100644 --- a/python-multiplatform-ksp/src/test/python/test_str_dict.py +++ b/python-multiplatform-ksp/src/test/python/test_str_dict.py @@ -26,70 +26,84 @@ import pytest +from conftest import decorate_compiled from typedpython import cbuild, cgen, frontend, verify REPO = Path(__file__).resolve().parents[4] WORDFREQ = REPO / "benchmarks" / "typedpython" / "py" / "wordfreq.py" SOURCE = textwrap.dedent(''' - # typedpython: compiled from typing import SupportsIndex + @compiled def get(d: dict[object, object], k: object) -> object: return d[k] + @compiled def get_list(xs: list[object], i: int) -> object: return xs[i] + @compiled def get_index(xs: list[object], i: SupportsIndex) -> object: return xs[i] + @compiled def get_str(s: str, i: int) -> str: return s[i] + @compiled def get_nested(d: dict[object, dict[object, object]], a: object, b: object) -> object: return d[a][b] + @compiled def put(d: dict[object, object], k: object, v: object) -> None: d[k] = v + @compiled def put_list(xs: list[object], i: int, v: object) -> None: xs[i] = v + @compiled def has(c: list[object], x: object) -> bool: return x in c + @compiled def has_not(c: list[object], x: object) -> bool: return x not in c + @compiled def has_dict(c: dict[object, object], x: object) -> bool: return x in c + @compiled def has_str(c: str, x: str) -> bool: return x in c + @compiled def has_branch(c: list[object], x: object) -> str: if x in c: return "yes" return "no" + @compiled def has_chain(c: list[object], x: object, y: object) -> bool: return x in c in y + @compiled def pairs(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b in xs: @@ -98,6 +112,7 @@ def pairs(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def triples(xs: list[tuple[object, ...]]) -> list[object]: out: list[object] = [] for a, b, c in xs: @@ -106,6 +121,7 @@ def triples(xs: list[tuple[object, ...]]) -> list[object]: return out + @compiled def each(xs: list[object]) -> list[object]: out: list[object] = [] for x in xs: @@ -115,6 +131,7 @@ def each(xs: list[object]) -> list[object]: return out + @compiled def first_after(xs: list[object], stop: object) -> object: found: object = None for x in xs: @@ -124,12 +141,14 @@ def first_after(xs: list[object], stop: object) -> object: return found + @compiled def return_inside(xs: list[object]) -> object: for x in xs: return x return None + @compiled def nested(xs: list[list[object]]) -> list[object]: out: list[object] = [] for row in xs: @@ -140,6 +159,7 @@ def nested(xs: list[list[object]]) -> list[object]: return out + @compiled def items_of(d: dict[str, object]) -> list[object]: out: list[object] = [] for k, v in d.items(): @@ -148,6 +168,7 @@ def items_of(d: dict[str, object]) -> list[object]: return out + @compiled def mutate_while_iterating(d: dict[str, object]) -> object: n: int = 0 for k in d: @@ -156,30 +177,37 @@ def mutate_while_iterating(d: dict[str, object]) -> object: return n + @compiled def fmt(x: object, y: float, n: int) -> str: return f"{x}|{x!r}|{x!s}|{x!a}|{y:.3f}|{n:05d}|{x:>5}|{n}" + @compiled def fmt_one(x: object) -> str: return f"{x}" + @compiled def fmt_text(x: object) -> str: return f"a{x}b{x!r}c" + @compiled def fmt_empty() -> str: return f"" + @compiled def fmt_debug(x: object) -> str: return f"{x=}" + @compiled def cat(a: str, b: str) -> str: return a + b + @compiled def cat_in_place(a: str, b: str) -> str: s: str = "" s += a @@ -187,6 +215,7 @@ def cat_in_place(a: str, b: str) -> str: return s + @compiled def grow(xs: list[object], ys: list[object]) -> list[object]: out: list[object] = [] out += xs @@ -194,6 +223,7 @@ def grow(xs: list[object], ys: list[object]) -> list[object]: return out + @compiled def empties() -> object: d: dict[object, object] = {} xs: list[object] = [] @@ -201,6 +231,7 @@ def empties() -> object: return xs + @compiled class Acc: def __init__(self, log: list[object]) -> None: self.log = log @@ -230,6 +261,7 @@ def __imod__(self, other: int) -> Acc: return self + @compiled def aug_ops(a: Acc, b: int) -> Acc: a += b a -= b @@ -240,12 +272,14 @@ def aug_ops(a: Acc, b: int) -> Acc: return a + @compiled def aug_list(xs: list[str], n: int) -> list[str]: xs *= n xs += xs return xs + @compiled def lookup_default(d: dict[str, int], k: str) -> object: counts: dict[str, int] = {} counts[k] = counts.get(k, 0) + 1 @@ -905,7 +939,7 @@ def run_main(main, n): @pytest.fixture(scope="module") def wordfreq(tmp_path_factory): d = tmp_path_factory.mktemp("wordfreq") - src = "# typedpython: compiled\n" + WORDFREQ.read_text() + src = decorate_compiled(WORDFREQ.read_text()) proved, diags, m = build_source(d, src, name="wordfreq") spec = importlib.util.spec_from_file_location("wordfreq_plain", WORDFREQ) plain = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/test/python/test_tracing.py b/python-multiplatform-ksp/src/test/python/test_tracing.py index 2c6939249..39f3c78b3 100644 --- a/python-multiplatform-ksp/src/test/python/test_tracing.py +++ b/python-multiplatform-ksp/src/test/python/test_tracing.py @@ -19,14 +19,15 @@ from typedpython import pipeline SOURCE = """\ -# typedpython: compiled from typing import Callable +@compiled def sq(x: float) -> float: return x * x +@compiled def loop(n: int) -> float: acc: float = 0.0 for k in range(n): @@ -34,11 +35,13 @@ def loop(n: int) -> float: return acc +@compiled def leaf(cb: Callable[[], None], n: float) -> float: cb() return n + 1.0 +@compiled def outer(cb: Callable[[], None], n: float) -> float: cb() return leaf(cb, n) + leaf(cb, n) From f702f48ff8a3f4f95a7cb086ce37c25b16836d58 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:58:51 +0900 Subject: [PATCH 05/61] TypedPython: no PyPI publishing; the wheel ships with the GitHub Release and the distribution Maintainer, 2026-10-30: the GitHub Release and the thisisthepy distribution are enough. Removes typedpython-pypi.yml; INTENT 1.4 says so. --- .github/workflows/typedpython-pypi.yml | 78 -------------------------- docs/INTENT.md | 6 +- 2 files changed, 3 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/typedpython-pypi.yml diff --git a/.github/workflows/typedpython-pypi.yml b/.github/workflows/typedpython-pypi.yml deleted file mode 100644 index b0f98705d..000000000 --- a/.github/workflows/typedpython-pypi.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: TypedPython to PyPI - -# Publishes the `typedpython` package (python-multiplatform-ksp/pyproject.toml) to PyPI. The package -# is versioned on its own, so it has its own tag: `typedpython-v`, pushed by the coordinator -# on a commit of main. PyPI trusted publishing: no token is stored; the `pypi` environment and the -# PyPI project's trusted publisher (this repository, this workflow file, environment `pypi`) must match. -# Only a wheel is published: it is pure Python (`py3-none-any`), and an sdist would carry the whole -# Kotlin module next to it. - -on: - push: - tags: - - 'typedpython-v*' - workflow_dispatch: - inputs: - tag: - description: 'Existing tag to build and publish (e.g. typedpython-v0.1.0)' - required: true - -permissions: - contents: read - -jobs: - build: - name: Build and check the wheel - runs-on: macos-latest - env: - TAG: ${{ github.event.inputs.tag || github.ref_name }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.tag || github.ref }} - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - - - name: The tag names the package version - run: | - set -euo pipefail - version="$(sed -nE 's/^version = "(.*)"$/\1/p' python-multiplatform-ksp/pyproject.toml | head -1)" - test "$TAG" = "typedpython-v$version" || { echo "error: tag $TAG does not match typedpython $version" >&2; exit 1; } - - - name: Build - run: uv build --wheel --out-dir dist python-multiplatform-ksp - - - name: The wheel alone compiles the release example - run: | - set -euo pipefail - uv venv -q -p 3.15t "$RUNNER_TEMP/tpenv" - uv pip install -q --python "$RUNNER_TEMP/tpenv/bin/python" dist/typedpython-*.whl - "$RUNNER_TEMP/tpenv/bin/typedpython" demo --out "$RUNNER_TEMP/tpbuild" docs/release/desktop/typedpython/kernels.py 400000 | tee "$RUNNER_TEMP/tp.log" - grep -q '^compiled functions: .*dot' "$RUNNER_TEMP/tp.log" - grep -q '^identical output' "$RUNNER_TEMP/tp.log" - - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - - publish: - name: Publish to PyPI - needs: build - runs-on: ubuntu-latest - environment: pypi - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist/ - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - - - name: Publish - run: uv publish --trusted-publishing always dist/* diff --git a/docs/INTENT.md b/docs/INTENT.md index 5c45079d1..b471a8573 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -41,9 +41,9 @@ CPython interpreter inside a Kotlin Multiplatform application so that: the embedded CPython, as an extension module (`docs/design/typedpython.md`). TypedPython reaches plain Python users as a **distribution**, not as a new interpreter (maintainer, - 2026-10-30: "배포판 방식을 생각했다는거야. 근데 일단 그렇게 하더라도 typedpython 패키지 형태라면 pypi - 배포를 해야겠지"). The thisisthepy CPython build ships with TypedPython in it, and the same - compiler is an ordinary `typedpython` package on PyPI (`uv add typedpython`). + 2026-10-30: "배포판 방식을 생각했다는거야"). The thisisthepy CPython build ships with TypedPython in + it. The `typedpython` wheel is attached to the GitHub Release and is not published to PyPI + ("그냥 github release로만 해도 충분한거라는거지 ... 그냥 우리 배포판으로 배포하면 되는거"). 5. **High-performance kernels are written in Python too** (TypedPython tier 2). The maintainer first asked to explore it ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자") and on From cca2ef5753f544a53c2c19eae5715e3a6b3a6616 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:01:16 +0900 Subject: [PATCH 06/61] TypedPython tests: child processes install the compiled builtin themselves Plain CPython without typedpython_builtins.pth has no builtins.compiled; the child scripts now prepend conftest's COMPILED_SHIM. Also fixes test_regions' split that left a dangling @compiled. --- python-multiplatform-ksp/src/test/python/conftest.py | 4 ++++ .../src/test/python/test_class_methods.py | 8 ++++---- .../src/test/python/test_eval_breaker.py | 3 ++- .../src/test/python/test_fast_entry.py | 3 ++- .../src/test/python/test_free_threaded.py | 3 ++- .../src/test/python/test_parallel_loops.py | 3 ++- .../src/test/python/test_recursion.py | 3 ++- python-multiplatform-ksp/src/test/python/test_regions.py | 4 +++- python-multiplatform-ksp/src/test/python/test_tracing.py | 3 ++- 9 files changed, 23 insertions(+), 11 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 669816bf4..85b038234 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -9,6 +9,10 @@ # The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +# Prepended to every child-process script that loads a module whose source uses `@compiled`: a plain +# CPython has the builtin only when `typedpython_builtins.pth` is installed, so the child installs it. +COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f: f\n" + FIXTURES = Path(__file__).parent / "fixtures" diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index cd08794e3..b35b5db6e 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -40,7 +40,7 @@ import pytest -from conftest import decorate_compiled +from conftest import COMPILED_SHIM, decorate_compiled from typedpython import cbuild, cgen, cost, frontend, pipeline, verify HERE = Path(__file__).resolve().parent @@ -553,7 +553,7 @@ def test_class_info_reports_the_methods(built): # --- class state changed after import: child processes -------------------------------------------- -CHILD = ''' +CHILD = COMPILED_SHIM + ''' import json, sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -736,7 +736,7 @@ def test_reference_counts_are_flat_over_many_calls(built): # --- free-threaded concurrency -------------------------------------------------------------------- -THREADS = ''' +THREADS = COMPILED_SHIM + ''' import sys, threading from pathlib import Path sys.path.insert(0, {main!r}) @@ -800,7 +800,7 @@ def test_concurrent_calls_on_shared_and_private_instances(built): # --- mutation: the exact-type guard of `self` ------------------------------------------------------- -GUARD_PROBE = ''' +GUARD_PROBE = COMPILED_SHIM + ''' import sys from pathlib import Path sys.path.insert(0, {main!r}) diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index 0d6bb643b..dece8d3b8 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -18,6 +18,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -49,7 +50,7 @@ def total(xs: list[float], rounds: int) -> float: return s """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, signal, sys, threading, time path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("loops", path) diff --git a/python-multiplatform-ksp/src/test/python/test_fast_entry.py b/python-multiplatform-ksp/src/test/python/test_fast_entry.py index d448e3b41..633a14487 100644 --- a/python-multiplatform-ksp/src/test/python/test_fast_entry.py +++ b/python-multiplatform-ksp/src/test/python/test_fast_entry.py @@ -20,6 +20,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -170,7 +171,7 @@ def test_the_fast_impl_is_never_addressed(traced): assert "&tp_fimpl_" not in c and "(tp_fimpl_" not in c -LOAD = """\ +LOAD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys def load(name, path, ext): if ext: diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index e954c03ba..e857d3950 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -27,6 +27,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, pipeline, verify STRESS = max(1, int(os.environ.get("TP_FT_STRESS", "1"))) @@ -155,7 +156,7 @@ def usek(x: int) -> int: return x + K ''' -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import faulthandler, importlib.machinery, importlib.util, json, sys, threading, time faulthandler.enable() path = sys.argv[1] diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index e86605b69..ecf4b8c7a 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -25,6 +25,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, frontend, ir, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -335,7 +336,7 @@ def test_list_untouched_when_the_call_raises_before_the_loop(mod): # --- child processes: thread counts, creation failure, concurrency ----------------------------------- -CHILD = """\ +CHILD = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, struct, sys, threading path = sys.argv[1] name = path.replace(chr(92), "/").rsplit("/", 1)[-1].split(".")[0] diff --git a/python-multiplatform-ksp/src/test/python/test_recursion.py b/python-multiplatform-ksp/src/test/python/test_recursion.py index 49901b0e6..84c36f1e6 100644 --- a/python-multiplatform-ksp/src/test/python/test_recursion.py +++ b/python-multiplatform-ksp/src/test/python/test_recursion.py @@ -15,6 +15,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" @@ -239,7 +240,7 @@ def inner(j): # --- sanitizers: the RecursionError path --------------------------------------------------------- -ASAN_SCRIPT = textwrap.dedent('''\ +ASAN_SCRIPT = COMPILED_SHIM + textwrap.dedent('''\ import sys sys.path.insert(0, {pkg!r}) from typedpython import cbuild diff --git a/python-multiplatform-ksp/src/test/python/test_regions.py b/python-multiplatform-ksp/src/test/python/test_regions.py index ac7aad6c8..7d7080ee9 100644 --- a/python-multiplatform-ksp/src/test/python/test_regions.py +++ b/python-multiplatform-ksp/src/test/python/test_regions.py @@ -25,6 +25,7 @@ import pytest from test_cgen import runtime_dir, need_runtime +from conftest import COMPILED_SHIM from typedpython import cbuild, cgen, cost, frontend, verify posix_signals = pytest.mark.skipif(sys.platform == "win32", reason="needs POSIX signals (SIGALRM/SIGUSR1/setitimer) or fork; Windows has none") @@ -392,10 +393,11 @@ def test_a_forged_module_level_variant_set_is_reported(built): def test_a_finalizer_or_a_field_store_keeps_the_class_out_of_a_region(tmp_path): - base = SOURCE.split("def escapes")[0] + base = SOURCE.split("@compiled\ndef escapes")[0] with_del = base.replace(" self.right = right\n", " self.right = right\n\n def __del__(self) -> None:\n pass\n", 1) with_store = base + textwrap.dedent('''\ + @compiled def poke(node: Node) -> int: node.left = None return 0 diff --git a/python-multiplatform-ksp/src/test/python/test_tracing.py b/python-multiplatform-ksp/src/test/python/test_tracing.py index 39f3c78b3..823c1dd1d 100644 --- a/python-multiplatform-ksp/src/test/python/test_tracing.py +++ b/python-multiplatform-ksp/src/test/python/test_tracing.py @@ -16,6 +16,7 @@ import pytest +from conftest import COMPILED_SHIM from typedpython import pipeline SOURCE = """\ @@ -47,7 +48,7 @@ def outer(cb: Callable[[], None], n: float) -> float: return leaf(cb, n) + leaf(cb, n) """ -LOADER = """\ +LOADER = COMPILED_SHIM + """\ import importlib.machinery, importlib.util, json, sys, threading path = sys.argv[1] loader = importlib.machinery.ExtensionFileLoader("tmod", path) From 0e54f8eeaee9c940491518b15551a3cc25bc5afa Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:49:18 +0900 Subject: [PATCH 07/61] Feat: replace # typedpython: parallel comment with @compiled(parallel=True) (#358) --- docs/SPEC.md | 4 +- docs/design/typedpython-parallel-loops.md | 15 +- .../src/main/python/typedpython/__main__.py | 2 +- .../builtin_stubs/__builtins__.pyi | 6 + .../src/main/python/typedpython/frontend.py | 85 ++++--- .../src/main/python/typedpython/ir.py | 4 +- .../src/main/python/typedpython_builtins.pth | 2 +- .../src/test/python/conftest.py | 4 +- .../src/test/python/test_builtin_compiled.py | 13 +- .../src/test/python/test_frontend.py | 23 ++ .../src/test/python/test_parallel_loops.py | 216 ++++++------------ .../python/multiplatform/ffi/Python3.kt | 4 +- .../multiplatform/ffi/BuiltinCompiledTest.kt | 4 +- 13 files changed, 192 insertions(+), 190 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 61c454c2f..a41dd5b2e 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -80,7 +80,7 @@ fail, then implement. extracts it. `Status: partial`, exercised by the shared suite on the simulator; no iOS-specific lifecycle test beyond `PM/iosSimulatorArm64Test/.../AsyncioAvailabilityProbeTest.kt`. An installed app carries its own stdlib: L-11. -- **L-8** Initialisation sets `builtins.compiled` to an identity decorator (an existing one is kept), so `@compiled` needs no import on any platform (N-7, issue #42). `Status: implemented`, `PM/commonTest/.../ffi/BuiltinCompiledTest.kt`. +- **L-8** Initialisation sets `builtins.compiled` to an identity decorator accepting both `@compiled` (`compiled(f)`) and `@compiled(**opts)` (`compiled(**opts)(f)`), leaving an existing `builtins.compiled` alone, so `@compiled` needs no import on any platform (N-7, issue #42, #358). `Status: implemented`, `PM/commonTest/.../ffi/BuiltinCompiledTest.kt`. - **L-9** Desktop: a packaged application starts with no `PYTHONHOME` in its environment (issue #60, ROADMAP §15.4). The Gradle plugin copies the prefix `stagePythonHome` stages into Compose Desktop's application resources (every `prepare*AppResources` task, so `createDistributable` and the @@ -985,7 +985,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **N-15** TypedPython parallel execution of order-independent loops on free-threaded CPython (#151; `docs/design/typedpython-parallel-loops.md`). A compiled `for ... in range(...)` loop may run its iterations on several threads only if its result cannot depend on iteration order, in a function the - user marked with a `# typedpython: parallel` comment line directly above the `def` (above its decorators if any); module-level functions only, and then the + user marked `@compiled(parallel=True)` (#358; the earlier `# typedpython: parallel` comment line was removed); module-level functions only, and then the result, the exception, the partial writes and the deopt behaviour are bit-for-bit those of the sequential loop (INTENT §1.4; N-12: every execution is indistinguishable from some valid CPython execution). diff --git a/docs/design/typedpython-parallel-loops.md b/docs/design/typedpython-parallel-loops.md index e86c3e06b..2834d69a6 100644 --- a/docs/design/typedpython-parallel-loops.md +++ b/docs/design/typedpython-parallel-loops.md @@ -30,8 +30,7 @@ checkable on the existing IR (`typedpython/ir.py`), and says where that rule giv (§7, decision 3). Thread creation cost is measured together with the threshold. 6. A cost rule decides, at run time, whether a region is worth its fixed cost; below the threshold the sequential code runs (§8). The threshold is measured, not guessed here. -7. v1 parallelises only functions the user marks (decision 5). The spelling of the marking is a design - detail settled at implementation. Widening to unmarked functions later is easy; narrowing is hard. +7. v1 parallelises only functions the user marks `@compiled(parallel=True)` (decision 5, #358). Widening to unmarked functions later is easy; narrowing is hard. ## 2. What exists today (the facts this design stands on) @@ -78,7 +77,7 @@ condition that failed (like `verify/array-open`), so users can see why. - The function holds a copy-in list array, a local array, or only scalars (a pure scalar loop, e.g. a reduction over `range`). A function whose loop touches a buffer array (`array.array`) is not parallelised (decision 1). -- The user marked the function (decision 5). +- The user marked the function `@compiled(parallel=True)` (decision 5, #358). - Not under `entry_globals` mutation: entry globals are read once at entry and are locals in the body, so they are read-only and fine. @@ -503,9 +502,11 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. measured together with the threshold (`TP_PAR_MIN_WORK`). 4. **Float `+` and `*` reductions are excluded in v1.** Not permanently: a fastmath opt-in (#153) awaits the owner's decision, and N-15 follows that decision. Until then they stay sequential. -5. **Only functions the user marks are parallelised in v1.** How the marking is spelled (decorator - argument, comment, other) is a design detail to settle at implementation. Widening later (implicit - parallelisation) is easy; narrowing later is hard, so v1 starts narrow. +5. **Only functions the user marks are parallelised in v1.** Marked with `@compiled(parallel=True)` + (maintainer decision 2026-10-30, #358: `@compiled` is the only compilation mark, and options are + keyword arguments on `@compiled`, matching `#332` `@compiled(fastmath=True)`; the earlier + `# typedpython: parallel` comment marker was removed). Widening later (implicit parallelisation) is + easy; narrowing later is hard, so v1 starts narrow. 6. **`min`/`max` reductions are in v1 only if exactly equal to Python's** (§6). If an implementation cannot match it, they move to a later step and v1 keeps the integer and bitwise reductions only. 7. **Thresholds are measured, not chosen here.** `TP_PAR_MIN_WORK` and the thread cap (default 8) are @@ -516,4 +517,4 @@ Recorded 2026-10-25 from the maintainer. The body and SPEC N-15 follow them. - (Settled: free-threaded 3.15t is the only build, #158, so there is no GIL-build variant.) - Which devices the threshold is measured on (the Mac mini only, or also an Android device). The measurement itself is decided (item 7); the device set is not. -- The spelling of the marking (item 5), settled at implementation. +- (Settled 2026-10-30, #358: the marking is `@compiled(parallel=True)`.) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index a9cda63e2..4db824787 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -113,7 +113,7 @@ def _demo(module: Path, module_argv: list[str], out: Path) -> int: def interpreted() -> object: import builtins # the module's `@compiled` is a builtin name; typedpython_builtins.pth installs it - builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) + builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) assert spec is not None and spec.loader is not None mod = importlib.util.module_from_spec(spec) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi index f773368b9..122e4cdf5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi +++ b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi @@ -1,3 +1,9 @@ # TypedPython's builtin names (docs/design/typedpython.md §2.3). A `__builtins__.pyi` on the type # checker's search path declares names the interpreter provides without an import. +from collections.abc import Callable +from typing import overload + +@overload def compiled[T](f: T, /) -> T: ... +@overload +def compiled[T](*, parallel: bool = ..., fastmath: bool = ...) -> Callable[[T], T]: ... diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 6e0b75aa6..a65ea1b9f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -190,7 +190,8 @@ def lower(path: Path) -> ir.Module: info.classes = class_decls skipped.update(class_skips) signatures: dict[str, _Signature] = {} - for node, reason in _targets(tree, source, info): + marked: set[str] = set() + for node, reason, parallel in _targets(tree, source, info): if reason is not None: skipped[node.name] = reason continue @@ -199,6 +200,9 @@ def lower(path: Path) -> ir.Module: signatures[node.name] = _signature(node, info.classes, info) except Skip as e: skipped[node.name] = e.reason + else: + if parallel: + marked.add(node.name) for owner, node, reason in _method_targets(tree, info): q = f"{owner}.{node.name}" if reason is not None: @@ -236,28 +240,10 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - marked = _parallel_marked(tree, source) functions = [dataclasses.replace(f, parallel=True) if f.name in marked else f for f in functions] return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) -PARALLEL_MARKER = "# typedpython: parallel" - - -def _parallel_marked(tree: ast.Module, source: str) -> set[str]: - """Module-level functions the user marked for SPEC N-15: the comment line `# typedpython: parallel` - directly above the `def` (above its decorators if it has any). A comment, - so the interpreted module is unchanged and needs no runtime name.""" - lines = source.splitlines() - out: set[str] = set() - for node in tree.body: - if isinstance(node, ast.FunctionDef): - first = min([node.lineno] + [d.lineno for d in node.decorator_list]) - if first >= 2 and lines[first - 2].strip() == PARALLEL_MARKER: - out.add(node.name) - return out - - def _module_info(tree: ast.Module) -> _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] = {} @@ -431,13 +417,45 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") +def _is_compiled_decorator(d: ast.expr) -> bool: + return (isinstance(d, ast.Name) and d.id == "compiled") or ( + isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == "compiled" + ) + + +def _compiled_decorator_options(d: ast.expr) -> tuple[bool, str | None]: + """Parse `@compiled` or `@compiled(parallel=...)` -> (parallel, error_reason_or_None).""" + if isinstance(d, ast.Name): + return False, None + assert isinstance(d, ast.Call) + if d.args: + return False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " + f"(got positional `{_src(d.args[0])}`)") + parallel = False + seen: set[str] = set() + for kw in d.keywords: + if kw.arg is None: + return False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" + if kw.arg != "parallel": + return False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " + "supported (only `parallel=True` or `parallel=False`)") + if kw.arg in seen: + return False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" + seen.add(kw.arg) + if not (isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, bool)): + return False, (f"line {d.lineno}: `@compiled(parallel=...)` requires a literal `True` " + f"or `False`, got `{_src(kw.value)}`") + parallel = kw.value.value + return parallel, None + + def is_opted_in(path: Path) -> bool: """Whether the front end considers the module for compilation at all: a module-level `def` / `class` decorated `@compiled` (the same test `_targets` and `_classes` apply).""" source = Path(path).read_text() return any( isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) - and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) + and any(_is_compiled_decorator(d) for d in node.decorator_list) for node in ast.parse(source, filename=str(path)).body) @@ -596,15 +614,16 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, -) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]]: + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue - marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] + marked = [d for d in node.decorator_list if _is_compiled_decorator(d)] if not marked: continue reason = None + parallel = False if marked and info.bound("compiled"): line = info.bindings["compiled"][0][0] reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " @@ -612,12 +631,20 @@ def _targets( elif other := [d for d in node.decorator_list if d not in marked]: reason = (f"line {other[0].lineno}: decorator `@{_src(other[0])}` is not supported " "(only `@compiled`)") - elif isinstance(node, ast.AsyncFunctionDef): - reason = f"line {node.lineno}: async function is not supported" - elif len(info.bindings.get(node.name, [])) > 1: - lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) - reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" - found.append((node, reason)) + else: + for d in marked: + par, r = _compiled_decorator_options(d) + if r is not None: + reason = r + break + parallel = parallel or par + if reason is None: + if isinstance(node, ast.AsyncFunctionDef): + reason = f"line {node.lineno}: async function is not supported" + elif len(info.bindings.get(node.name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) + reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" + found.append((node, reason, parallel)) return found diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 368c1c3ba..1176d2182 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -817,8 +817,8 @@ class Function: # rejects a function whose declared entries differ (`verify/virtual-escape`). The front end # leaves it empty. virtual: tuple[str, ...] = () - # SPEC N-15: the user marked this function with the comment line `# typedpython: parallel` - # directly above its `def`. Only a marked function can hold a `ForRange.par`. + # SPEC N-15: the user marked this function `@compiled(parallel=True)` (#358). + # Only a marked function can hold a `ForRange.par`. parallel: bool = False # SPEC N-12b (#268): the producers whose result this pure function keeps inside one statement # (`check(make(d))`), so each such call builds its objects in a region. Set only by `verify`, diff --git a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth index 98776970a..1b8b0a0ad 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth +++ b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth @@ -1 +1 @@ -import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 85b038234..975fdc08b 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -7,11 +7,11 @@ import pytest # The `typedpython_builtins.pth` of an installed wheel; the test interpreter has no such install. -builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) +builtins.compiled = getattr(builtins, "compiled", lambda f=None, /, **_k: (lambda g: g) if f is None else f) # Prepended to every child-process script that loads a module whose source uses `@compiled`: a plain # CPython has the builtin only when `typedpython_builtins.pth` is installed, so the child installs it. -COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f: f\n" +COMPILED_SHIM = "import builtins\nif not hasattr(builtins, 'compiled'):\n builtins.compiled = lambda f=None, /, **_k: (lambda g: g) if f is None else f\n" FIXTURES = Path(__file__).parent / "fixtures" diff --git a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py index 4f184812b..1357c7ed7 100644 --- a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py +++ b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py @@ -21,7 +21,16 @@ def test_gate_accepts_compiled_without_an_import(write): def f(x: int) -> int: return x + 1 - y: int = f(1) + @compiled(parallel=True) + def g(x: int) -> int: + return x + 2 + + @compiled(fastmath=True) + def h(x: float) -> float: + return x * 2.0 + + y: int = f(1) + g(2) + z: float = h(1.5) """) assert errors(gate.check([path])) == [] @@ -82,7 +91,7 @@ def test_decorator_works_with_no_import(installed): def test_compiled_is_the_identity(installed): python, _ = installed - r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f)") + r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f and compiled(parallel=True)(f) is f and compiled()(f) is f)") assert r.stdout.strip() == "True", r.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index fe8f4d94c..ddcf3fcc9 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -654,6 +654,29 @@ def c(x: float) -> float: assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] +def test_compiled_parallel_option_and_former_comment_marker(tmp_path): + m = lower(tmp_path, """ + @compiled(parallel=True) + def p(x: float) -> float: + return x + + @compiled(parallel=False) + def s(x: float) -> float: + return -x + + # typedpython: parallel + @compiled + def c(x: float) -> float: + return x * 2.0 + + @compiled(unknown=True) + def bad(x: float) -> float: + return x + """) + assert [(f.name, f.parallel) for f in m.functions] == [("p", True), ("s", False), ("c", False)] + assert set(m.skipped) == {"bad"} and "parallel" in m.skipped["bad"] + + def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): m = lower(tmp_path, """ # typedpython: compiled diff --git a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py index ecf4b8c7a..8f215ecd2 100644 --- a/python-multiplatform-ksp/src/test/python/test_parallel_loops.py +++ b/python-multiplatform-ksp/src/test/python/test_parallel_loops.py @@ -1,14 +1,14 @@ -"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277). +"""TypedPython parallel order-independent loops, step 1 (SPEC N-15, #151, #277, #358). -Step 1 scope: in a function marked by the comment line `# typedpython: parallel` directly above its -`def`, a `for i in range(len(a))` loop whose body is only stores `a[i] = ` (the same +Step 1 scope: in a function decorated with `@compiled(parallel=True)`, a `for i in range(len(a))` +loop whose body is only stores `a[i] = ` (the same index i, each proven in bounds), over list copy-in or local arrays, with values built from constants, loop-invariant scalars, `float(i)`, reads `b[i]` at the same i (proven) and F64 + - * and negation, runs on several C threads above TP_PAR_MIN_WORK. The verifier recomputes the set (`ForRange.par`) and rejects a declared one that differs (`verify/par-loop`). Everything else compiles as before. Red record: written before the implementation. Before it, `ir.ForRange` had no `par`, `ir.Function` -had no `parallel`, the marker comment was ignored and the generated module had no +had no `parallel`, the parallel opt-in was ignored and the generated module had no `__typedpython_parallel__`; every test below that touches those failed (see the report in #277). """ from __future__ import annotations @@ -35,8 +35,7 @@ SOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b @@ -48,30 +47,26 @@ def scale_unmarked(xs: list[float], a: float, b: float) -> None: xs[i] = xs[i] * a + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def ramp(xs: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = -(float(i) * a) + xs[i] -# typedpython: parallel -@compiled +@compiled(parallel=True) def two_stores(xs: list[float], ys: list[float], a: float) -> None: for i in range(len(xs)): xs[i] = xs[i] + a xs[i] = xs[i] * xs[i] -# typedpython: parallel -@compiled +@compiled(parallel=True) def neighbour(xs: list[float]) -> None: for i in range(len(xs) - 1): xs[i] = xs[i + 1] -# typedpython: parallel -@compiled +@compiled(parallel=True) def total(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -84,22 +79,19 @@ def helper(x: float) -> float: return x * 2.0 -# typedpython: parallel -@compiled +@compiled(parallel=True) def with_call(xs: list[float]) -> None: for i in range(len(xs)): xs[i] = helper(xs[i]) -# typedpython: parallel -@compiled +@compiled(parallel=True) def divides(xs: list[float], d: float) -> None: for i in range(len(xs)): xs[i] = xs[i] / d -# typedpython: parallel -@compiled +@compiled(parallel=True) def local_fill(a: float) -> float: t = [0.0] * 300000 for i in range(len(t)): @@ -107,8 +99,7 @@ def local_fill(a: float) -> float: return t[7] + t[299999] -# typedpython: parallel -@compiled +@compiled(parallel=True) def local_dyn(n: int, a: float) -> float: t = [1.0] * n for i in range(len(t)): @@ -250,7 +241,7 @@ def test_c_has_a_region_only_for_eligible_loops(built): def test_no_par_code_without_a_marker(tmp_path): - src = SOURCE.replace("# typedpython: parallel\n", "") + src = SOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="nomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -450,8 +441,7 @@ def sh(): RSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -459,8 +449,7 @@ def r_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum_init(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -468,8 +457,7 @@ def r_sum_init(xs: list[int], s0: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -477,8 +465,7 @@ def r_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_sum_mul(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -486,8 +473,7 @@ def r_sum_mul(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_two(xs: list[int]) -> int: a: int = 0 b: int = 7 @@ -497,8 +483,7 @@ def r_two(xs: list[int]) -> int: return a * 3 + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_range_sum(xs: list[int]) -> int: s: int = 5 for i in range(len(xs)): @@ -506,8 +491,7 @@ def r_range_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def r_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -526,8 +510,7 @@ def r_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -535,8 +518,7 @@ def n_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -545,8 +527,7 @@ def n_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_float(xs: list[float]) -> float: s: float = 0.0 for i in range(len(xs)): @@ -554,8 +535,7 @@ def n_float(xs: list[float]) -> float: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -565,8 +545,7 @@ def n_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_mixed(xs: list[int]) -> int: t = [0.0] * len(xs) s: int = 0 @@ -576,8 +555,7 @@ def n_mixed(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -585,8 +563,7 @@ def n_div(xs: list[int], d: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_shift_step(xs: list[int]) -> int: s: int = 0 for i in range(0, len(xs), 2): @@ -864,7 +841,7 @@ def test_reduction_results_identical_for_every_thread_count(rbuilt): def test_no_reduction_region_without_a_marker(tmp_path): - src = RSOURCE.replace("# typedpython: parallel\n", "") + src = RSOURCE.replace("@compiled(parallel=True)", "@compiled") _, c_text, _, _ = build(tmp_path, src, name="rnomark") assert "tp_par_run(" not in c_text and "__typedpython_parallel__" not in c_text @@ -885,8 +862,7 @@ def test_no_reduction_region_without_a_marker(tmp_path): MSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -895,8 +871,7 @@ def mn_i(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_i(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -905,8 +880,7 @@ def mx_i(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -915,8 +889,7 @@ def mn_f(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_f(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -925,8 +898,7 @@ def mx_f(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -935,8 +907,7 @@ def mn_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mx_f_flipped(xs: list[float], init: float) -> float: m = init for i in range(len(xs)): @@ -945,8 +916,7 @@ def mx_f_flipped(xs: list[float], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_expr(xs: list[int], c: int) -> int: m = c for i in range(len(xs)): @@ -955,8 +925,7 @@ def mn_i_expr(xs: list[int], c: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_f_expr(xs: list[float], a: float) -> float: m = a for i in range(len(xs)): @@ -965,8 +934,7 @@ def mn_f_expr(xs: list[float], a: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_and_sum(xs: list[int]) -> int: m = xs[0] s = 0 @@ -977,8 +945,7 @@ def mn_and_sum(xs: list[int]) -> int: return m * 1000003 + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def two_ext(xs: list[float]) -> float: lo = xs[0] hi = xs[0] @@ -990,8 +957,7 @@ def two_ext(xs: list[float]) -> float: return hi - lo -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_init(xs: list[int], init: int) -> int: m = init for i in range(len(xs)): @@ -1000,8 +966,7 @@ def mn_i_init(xs: list[int], init: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def mn_i_big_e(xs: list[int], c: int) -> int: m = 0 for i in range(len(xs)): @@ -1019,8 +984,7 @@ def mn_unmarked(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_le(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1029,8 +993,7 @@ def n_le(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_other_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1039,8 +1002,7 @@ def n_other_value(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_else(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1052,8 +1014,7 @@ def n_else(xs: list[int]) -> int: return m + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_two_in_then(xs: list[int]) -> int: m = xs[0] s = 0 @@ -1064,8 +1025,7 @@ def n_two_in_then(xs: list[int]) -> int: return m + s -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_in_value(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1075,8 +1035,7 @@ def n_acc_in_value(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_cond_other(xs: list[int], t: int) -> int: m = xs[0] for i in range(len(xs)): @@ -1085,8 +1044,7 @@ def n_cond_other(xs: list[int], t: int) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_mixed_types(xs: list[int], init: float) -> float: m = init for i in range(len(xs)): @@ -1095,8 +1053,7 @@ def n_mixed_types(xs: list[int], init: float) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_acc_reads_acc(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1107,8 +1064,7 @@ def n_acc_reads_acc(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def n_div(xs: list[float], d: float) -> float: m = xs[0] for i in range(len(xs)): @@ -1537,8 +1493,7 @@ def spin(): BSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1546,8 +1501,7 @@ def b_and(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1555,8 +1509,7 @@ def b_or(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1564,8 +1517,7 @@ def b_xor(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_and_swapped(xs: list[int], s0: int) -> int: s: int = s0 for i in range(len(xs)): @@ -1573,8 +1525,7 @@ def b_and_swapped(xs: list[int], s0: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or_swapped(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1582,8 +1533,7 @@ def b_or_swapped(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_xor_expr(xs: list[int], c: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1591,8 +1541,7 @@ def b_xor_expr(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_or_not(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1600,8 +1549,7 @@ def b_or_not(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_mix(xs: list[int]) -> int: a: int = 0 x: int = 5 @@ -1616,8 +1564,7 @@ def b_mix(xs: list[int]) -> int: return a + x * 3 + o * 5 + m * 7 -# typedpython: parallel -@compiled +@compiled(parallel=True) def b_local(n: int) -> int: t = [0] * n for i in range(len(t)): @@ -1636,8 +1583,7 @@ def b_unmarked(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_acc_read(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1645,8 +1591,7 @@ def nb_acc_read(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_two_ops(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1655,8 +1600,7 @@ def nb_two_ops(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_and_then_add(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1665,8 +1609,7 @@ def nb_and_then_add(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_shift(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1674,8 +1617,7 @@ def nb_shift(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_rshift(xs: list[int], k: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1683,8 +1625,7 @@ def nb_rshift(xs: list[int], k: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_copy(xs: list[int]) -> int: s: int = 0 w: int = 0 @@ -1694,8 +1635,7 @@ def nb_copy(xs: list[int]) -> int: return s + w -# typedpython: parallel -@compiled +@compiled(parallel=True) def nb_div(xs: list[int], d: int) -> int: s: int = 0 for i in range(len(xs)): @@ -1857,15 +1797,13 @@ def test_bitwise_equals_interpreter_for_every_thread_count(bbuilt, threads): TSOURCE = '''\ -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_scale(xs: list[float], a: float, b: float) -> None: for i in range(len(xs)): xs[i] = xs[i] * a + b -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_sum(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1873,8 +1811,7 @@ def t_sum(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_prod(xs: list[int], c: int) -> int: s: int = 1 for i in range(len(xs)): @@ -1882,8 +1819,7 @@ def t_prod(xs: list[int], c: int) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_min(xs: list[int]) -> int: m = xs[0] for i in range(len(xs)): @@ -1892,8 +1828,7 @@ def t_min(xs: list[int]) -> int: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_max(xs: list[float]) -> float: m = xs[0] for i in range(len(xs)): @@ -1902,8 +1837,7 @@ def t_max(xs: list[float]) -> float: return m -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_and(xs: list[int]) -> int: s: int = -1 for i in range(len(xs)): @@ -1911,8 +1845,7 @@ def t_and(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_or(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): @@ -1920,8 +1853,7 @@ def t_or(xs: list[int]) -> int: return s -# typedpython: parallel -@compiled +@compiled(parallel=True) def t_xor(xs: list[int]) -> int: s: int = 0 for i in range(len(xs)): diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index 14d153326..5060b9469 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -124,7 +124,9 @@ object Python3 { def _pm_install_compiled(): import builtins if not hasattr(builtins, "compiled"): - def compiled(f): + def compiled(f=None, /, **_opts): + if f is None: + return lambda g: g return f builtins.compiled = compiled _pm_install_compiled() diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt index 5d3ef407f..e7765283f 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/BuiltinCompiledTest.kt @@ -11,7 +11,9 @@ class BuiltinCompiledTest { @Test fun compiledResolvesWithoutImportAndIsIdentity() = PythonTestFixture.withInterpreter { assertEquals("True", evalStr("compiled(len) is len")) - Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f") + assertEquals("True", evalStr("compiled(parallel=True)(len) is len")) + assertEquals("True", evalStr("compiled()(len) is len")) + Python3.exec("def _bc_f(): return 1\n_bc_r = compiled(_bc_f) is _bc_f and compiled(parallel=True)(_bc_f) is _bc_f") assertEquals("True", evalStr("_bc_r")) // It must live in builtins, not leak into __main__. assertEquals("False", evalStr("'compiled' in globals()")) From 88243859f91849d45471b3d8e53c66ffe15cc7dd Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:59:03 +0900 Subject: [PATCH 08/61] Feat: add per-function @compiled(fastmath=True) opt-in (#332) --- docs/SPEC.md | 9 ++++ .../src/main/python/typedpython/cgen.py | 33 ++++++++++-- .../src/main/python/typedpython/frontend.py | 53 ++++++++++++------- .../src/main/python/typedpython/ir.py | 3 ++ .../src/test/python/test_frontend.py | 37 +++++++++++++ .../src/test/python/test_pipeline.py | 53 +++++++++++++++++++ 6 files changed, 165 insertions(+), 23 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index a41dd5b2e..1e0697a64 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -739,6 +739,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `continue` and `return` meaning exactly what they mean without chunking (trip counts just below, at and above a multiple of the interval, negative steps, empty ranges and ranges spanning the i64 limits all behave as CPython's `range`). A function holding an array copy-in or entry-globals snapshot polls too (#152, option (i)): at the top of a loop iteration, once per `TP_POLL_INTERVAL`, it writes its copied lists back (CPython's own stores would already be visible there), ends their critical sections, polls, takes the critical sections again, copies the lists in again and re-checks the entry guards (list lengths, exact element types, entry globals). Unchanged: it continues compiled, which is an execution CPython can produce, since every list operation of another thread now falls before or after this point. Changed: it leaves for the interpreter at that point (an OSR-style exit) and does not deopt or raise: a continuation function generated from the function's own source, holding the rest of the function from that loop iteration (the innermost loop from its current index; each enclosing loop's remaining body, then its remaining iterations; then the statements after), receives every live local (a native value as its Python object, a local array as a new list) and each enclosing `range`'s already evaluated bounds, and the compiled call returns what it returns or raises what it raises. The continuation keeps the original file, line numbers and function name, so a traceback reads as the interpreted function's. + **Floating-point exactness and `@compiled(fastmath=True)` (#153, #332, owner decision 2026-10-25):** + by default (`fastmath=False`), floating-point compilation is bit-identical to CPython (`#pragma STDC + FP_CONTRACT OFF` in the generated C, `-ffp-contract=off -fno-fast-math` on GCC/Clang, `/fp:precise` + on MSVC). A module-level function marked `@compiled(fastmath=True)` opts into floating-point + reassociation and FMA contraction for that function only (never on by default, never applied to an + unmarked function or another function in the same module, and never inlined into a non-fastmath + caller without fastmath): **this is the one path in TypedPython that is explicitly NOT + CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round + differently from the interpreter so float reductions and FMA loops can vectorise). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index fb776ce01..940837ed9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -494,10 +494,19 @@ def _runs_no_python(f: ir.Function) -> bool: def _bounded_functions(functions: dict[str, ir.Function]) -> dict[str, int]: """name -> k for every bounded function (module docstring, "Fast entry"): in no call cycle, every callee bounded, `_runs_no_python`. k(f) = 1 + max(k(callee)), 1 without callees: the most - compiled frames a call of `f` can have in flight at once.""" + compiled frames a call of `f` can have in flight at once. + A `fastmath` function is never given an `inline` fast twin (`tp_fimpl_*`) when some other + function in the module calls it without `fastmath`, so a non-fastmath caller can never inline + a fastmath body (#332).""" callees: dict[str, set[str]] = {} for name, f in functions.items(): callees[name] = {e.function for e in _all_exprs(f) if isinstance(e, ir.Call)} + called_from_exact = { + c + for caller_name, cs in callees.items() + if not functions[caller_name].fastmath + for c in cs + } k: dict[str, int | None] = {} # None: not bounded (also while on the DFS stack) def visit(name: str) -> int | None: @@ -505,7 +514,7 @@ def visit(name: str) -> int | None: return k[name] k[name] = None # on the stack: reaching it again is a cycle f = functions[name] - if not _runs_no_python(f): + if not _runs_no_python(f) or (f.fastmath and name in called_from_exact): return None deepest = 0 for c in callees[name]: @@ -743,6 +752,21 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#if !defined(_MSC_VER) || defined(__clang__) /* cl.exe warns on an unknown pragma */") w("#pragma STDC FP_CONTRACT OFF") w("#endif") + if any(f.fastmath for f in module.functions): + w("/* SPEC N-8 (#153, #332): per-function `@compiled(fastmath=True)` opt-in (NOT CPython-identical). */") + w("#if defined(__clang__)") + w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") + w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") + w("#elif defined(__GNUC__)") + w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(\\\"fast-math\\\")\")") + w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") + w("#elif defined(_MSC_VER)") + w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") + w("#define TP_FASTMATH_END __pragma(fp_contract(off)) __pragma(float_control(pop))") + w("#else") + w("#define TP_FASTMATH_BEGIN") + w("#define TP_FASTMATH_END") + w("#endif") w("") w(f"static const char tp_source[] =\n {c_source_init(source)};") w(f"static const char tp_source_path[] = {c_string(display_path)};") @@ -2892,7 +2916,8 @@ def generate(self) -> str: """The impl (preceded by its fast twin when bounded) and the wrapper.""" parts = [] if self.rmode: - return self.impl() + text = self.impl() + return f"TP_FASTMATH_BEGIN\n{text}TP_FASTMATH_END\n" if self.f.fastmath else text if self.f.name in self.mg.bounded: twin = _FnGen(self.mg, self.f, fast=True) twin_text = twin.impl() @@ -2901,6 +2926,8 @@ def generate(self) -> str: impl_text = self.impl() parts += self.par_defs parts.append(impl_text) + if self.f.fastmath: + parts = ["TP_FASTMATH_BEGIN", *parts, "TP_FASTMATH_END"] parts.append(self.wrapper()) return "\n".join(parts) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index a65ea1b9f..83fa2083d 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -191,7 +191,8 @@ def lower(path: Path) -> ir.Module: skipped.update(class_skips) signatures: dict[str, _Signature] = {} marked: set[str] = set() - for node, reason, parallel in _targets(tree, source, info): + fastmath_marked: set[str] = set() + for node, reason, parallel, fastmath in _targets(tree, source, info): if reason is not None: skipped[node.name] = reason continue @@ -203,6 +204,8 @@ def lower(path: Path) -> ir.Module: else: if parallel: marked.add(node.name) + if fastmath: + fastmath_marked.add(node.name) for owner, node, reason in _method_targets(tree, info): q = f"{owner}.{node.name}" if reason is not None: @@ -240,7 +243,11 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - functions = [dataclasses.replace(f, parallel=True) if f.name in marked else f for f in functions] + functions = [ + dataclasses.replace(f, parallel=f.name in marked, fastmath=f.name in fastmath_marked) + if (f.name in marked or f.name in fastmath_marked) else f + for f in functions + ] return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) @@ -423,30 +430,34 @@ def _is_compiled_decorator(d: ast.expr) -> bool: ) -def _compiled_decorator_options(d: ast.expr) -> tuple[bool, str | None]: - """Parse `@compiled` or `@compiled(parallel=...)` -> (parallel, error_reason_or_None).""" +def _compiled_decorator_options(d: ast.expr) -> tuple[bool, bool, str | None]: + """Parse `@compiled` or `@compiled(parallel=..., fastmath=...)` -> (parallel, fastmath, error_reason_or_None).""" if isinstance(d, ast.Name): - return False, None + return False, False, None assert isinstance(d, ast.Call) if d.args: - return False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " - f"(got positional `{_src(d.args[0])}`)") + return False, False, (f"line {d.lineno}: `@compiled` takes keyword arguments only " + f"(got positional `{_src(d.args[0])}`)") parallel = False + fastmath = False seen: set[str] = set() for kw in d.keywords: if kw.arg is None: - return False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" - if kw.arg != "parallel": - return False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " - "supported (only `parallel=True` or `parallel=False`)") + return False, False, f"line {d.lineno}: `**{_src(kw.value)}` in `@compiled(...)` is not supported" + if kw.arg not in ("parallel", "fastmath"): + return False, False, (f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is not " + "supported (only `parallel` and `fastmath` with literal `True` or `False`)") if kw.arg in seen: - return False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" + return False, False, f"line {d.lineno}: `@compiled` keyword argument `{kw.arg}` is repeated" seen.add(kw.arg) if not (isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, bool)): - return False, (f"line {d.lineno}: `@compiled(parallel=...)` requires a literal `True` " - f"or `False`, got `{_src(kw.value)}`") - parallel = kw.value.value - return parallel, None + return False, False, (f"line {d.lineno}: `@compiled({kw.arg}=...)` requires a literal `True` " + f"or `False`, got `{_src(kw.value)}`") + if kw.arg == "parallel": + parallel = kw.value.value + else: + fastmath = kw.value.value + return parallel, fastmath, None def is_opted_in(path: Path) -> bool: @@ -614,8 +625,8 @@ def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | def _targets( tree: ast.Module, source: str, info: _ModuleInfo, -) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]]: - found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool]] = [] +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]]: + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None, bool, bool]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue @@ -624,6 +635,7 @@ def _targets( continue reason = None parallel = False + fastmath = False if marked and info.bound("compiled"): line = info.bindings["compiled"][0][0] reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " @@ -633,18 +645,19 @@ def _targets( "(only `@compiled`)") else: for d in marked: - par, r = _compiled_decorator_options(d) + par, fm, r = _compiled_decorator_options(d) if r is not None: reason = r break parallel = parallel or par + fastmath = fastmath or fm if reason is None: if isinstance(node, ast.AsyncFunctionDef): reason = f"line {node.lineno}: async function is not supported" elif len(info.bindings.get(node.name, [])) > 1: lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" - found.append((node, reason, parallel)) + found.append((node, reason, parallel, fastmath)) return found diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 1176d2182..4b28c068f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -820,6 +820,9 @@ class Function: # SPEC N-15: the user marked this function `@compiled(parallel=True)` (#358). # Only a marked function can hold a `ForRange.par`. parallel: bool = False + # SPEC N-8 (#153, #332): the user marked this function `@compiled(fastmath=True)` (reassociation + # and FMA contraction enabled for this function only; NOT CPython-identical). + fastmath: bool = False # SPEC N-12b (#268): the producers whose result this pure function keeps inside one statement # (`check(make(d))`), so each such call builds its objects in a region. Set only by `verify`, # which recomputes it and rejects a function whose declared entries differ diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index ddcf3fcc9..cd631efa6 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -677,6 +677,43 @@ def bad(x: float) -> float: assert set(m.skipped) == {"bad"} and "parallel" in m.skipped["bad"] +def test_compiled_fastmath_option_and_validation(tmp_path): + m = lower(tmp_path, """ + @compiled(fastmath=True) + def fm(x: float) -> float: + return x + + @compiled(fastmath=False) + def exact(x: float) -> float: + return -x + + @compiled(parallel=True, fastmath=True) + def both(x: float) -> float: + return x * 2.0 + + @compiled + def plain(x: float) -> float: + return x + 1.0 + + @compiled(fastmath=1) + def bad_val(x: float) -> float: + return x + + @compiled(other=True) + def bad_kw(x: float) -> float: + return x + """) + assert [(f.name, f.parallel, f.fastmath) for f in m.functions] == [ + ("fm", False, True), + ("exact", False, False), + ("both", True, True), + ("plain", False, False), + ] + assert set(m.skipped) == {"bad_val", "bad_kw"} + assert "fastmath" in m.skipped["bad_val"] + assert "fastmath" in m.skipped["bad_kw"] and "parallel" in m.skipped["bad_kw"] + + def test_the_former_module_pragma_comment_compiles_nothing(tmp_path): m = lower(tmp_path, """ # typedpython: compiled diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index ed326eb4c..18355aec4 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -185,3 +185,56 @@ def test_a_fresh_local_does_not_exempt_the_callee_class_guard(tmp_path): c = mod.Cell(None, None) mod.seta(c, 7) assert c.a == 7 + + +def test_default_fp_flags_stay_exact_and_fastmath_is_per_function_only(write, tmp_path): + """#153 / #332: by default, `-ffp-contract=off` and `-fno-fast-math` on GCC/Clang and + `/fp:precise` on MSVC are pinned so floating-point arithmetic is bit-identical to CPython. + `@compiled(fastmath=True)` opts in for that function only (not CPython-identical), leaving + sibling `@compiled` functions in the same module exact.""" + from typedpython import cbuild, cgen, frontend, verify + + assert "-ffp-contract=off" in cbuild.DEFAULT_FLAGS + assert "-fno-fast-math" in cbuild.DEFAULT_FLAGS + assert "/fp:precise" in cbuild.MSVC_DEFAULT_FLAGS + assert not any("fast" in f for f in cbuild.MSVC_DEFAULT_FLAGS) + + path = write( + "fp_mod.py", + """ + @compiled + def fma_exact(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled(fastmath=True) + def fma_fast(a: float, b: float, c: float) -> float: + return a * b + c + + @compiled + def calls_fast_and_adds(a: float, b: float, c: float) -> float: + # caller is exact: its own `a * b + c` must not contract even though it calls `fma_fast` + _ = fma_fast(1.0, 2.0, 3.0) + return a * b + c + """, + ) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + assert "#pragma STDC FP_CONTRACT OFF" in c_text + assert "\nTP_FASTMATH_BEGIN\n" in c_text and "\nTP_FASTMATH_END\n" in c_text + # Only `fma_fast` is wrapped in standalone TP_FASTMATH_BEGIN / TP_FASTMATH_END lines: + assert c_text.count("\nTP_FASTMATH_BEGIN\n") == 1 + assert c_text.count("\nTP_FASTMATH_END\n") == 1 + + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + mod = pipeline.load(res) + ref = interpreted(path) + # 1 + 2**-27 squared: exact double product rounds to 1 + 2**-26, while FMA(a, a, -1.0) keeps 2**-54 + a = 1.0 + 2.0**-27 + c = -1.0 + want_exact = ref.fma_exact(a, a, c) + assert mod.fma_exact(a, a, c) == want_exact + assert mod.calls_fast_and_adds(a, a, c) == want_exact + # fma_fast runs and returns a float (on targets with FMA it equals 2**-26 + 2**-54; either is valid under fastmath) + got_fast = mod.fma_fast(a, a, c) + assert isinstance(got_fast, float) + From 2b1bac3a3d623f96acbf3a2ae5c30c30937e527a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:32:50 +0900 Subject: [PATCH 09/61] TypedPython: TSan test ignores races against new_threadstate reusing a freed block Linux CI (run 41212647289) reported tp_traced's read of an exited thread's PyThreadState racing with libpython's new_threadstate memcpy into the same reused mimalloc block; free and reuse are ordered inside the uninstrumented allocator, as in #347. --- .../src/test/python/test_free_threaded.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index e857d3950..a1e925f04 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -840,12 +840,14 @@ def _access_is_ours(block: str) -> bool: A report where one access is mimalloc initialising a fresh block (`_mi_malloc*` memset) is not ours either: the extension's earlier read hit memory a thread later freed (an exited thread's `PyThreadState`, read by `tp_traced`), and the free and the reuse both happen inside the - uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI).""" + uninstrumented allocator, so TSan cannot see that they are ordered (seen on Linux CI). The same + holds when the reused block is filled by libpython's `new_threadstate` (a memcpy of the template + thread state into the block an exited thread's state was freed from, seen on Linux CI).""" firsts = [] for part in block.split("\n\n")[:2]: frames = [ln for ln in part.splitlines() if ln.strip().startswith("#")] firsts.append(next((f for f in frames if "libtsan" not in f and "sanitizer_common" not in f), "")) - if any(" _mi_malloc" in f or " _mi_page_malloc" in f for f in firsts): + if any(" _mi_malloc" in f or " _mi_page_malloc" in f or " new_threadstate " in f for f in firsts): return False return any("ftmod" in f for f in firsts) From 371e9679ed7f1c6709164e69cd871ec4ec203d2a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:53:02 +0900 Subject: [PATCH 10/61] Test: Make mid() loop heavier in test_eval_breaker to avoid CI timing flake (#280) --- .../src/test/python/test_eval_breaker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index dece8d3b8..faea4a5ae 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -397,7 +397,10 @@ def test_keyboard_interrupt_reaches_a_long_range_loop_in_chunks(extension): def mid(n: int) -> float: x: float = 1.0 for k in range(n): - x = x * 0.9999999 + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 1.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 2.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 3.0) + x = (x + 1.0) / (x + 2.0) + 1.0 / (k + 4.0) return x """ @@ -419,7 +422,7 @@ def handler(signum, frame): t0 = time.perf_counter() m.mid(n) full = time.perf_counter() - t0 - signal.setitimer(signal.ITIMER_REAL, full / 10) + signal.setitimer(signal.ITIMER_REAL, full / 20) t0 = time.perf_counter() try: m.mid(n) @@ -430,4 +433,5 @@ def handler(signum, frame): print(json.dumps({"out": out, "full": full, "elapsed": time.perf_counter() - t0})) """) assert r["out"] == "interrupted", r + assert r["full"] > 0.1, r assert r["elapsed"] < r["full"] * 0.8, r # before #280 the signal waited for the whole loop From c14d26498ffeae54a9bf387bfbc6b6a524c97a4d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:08:49 +0900 Subject: [PATCH 11/61] Feat: Switch notebook-e2e native-image CI to Maven Central and stage sample native-image prefix (#191, #251) --- .github/workflows/desktop.yml | 36 +--- docs/SPEC.md | 2 +- .../fork-desktop-paths-macos.md | 196 +++++++----------- .../fixtures/notebook-e2e/README.md | 5 +- sample/build.gradle.kts | 12 +- 5 files changed, 96 insertions(+), 155 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 0b43ff3e5..3956921f1 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -151,26 +151,15 @@ jobs: python-multiplatform-compose/fixtures/notebook-e2e/build/reports/tests/desktopTest python-multiplatform-compose/fixtures/notebook-e2e/build/test-results/desktopTest - # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes `ui`, `ui-text` and - # `foundation` from compose-multiplatform-core-extended, which a runner reads from the fork's GitHub - # Packages registry; until the repository variable COMPOSE_EXTENDED_VERSION names a published - # development version (`1.11.1-ext.N-dev.`) the job is skipped. + # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes the fork's Compose + # (`org.thisisthepy.compose.*:1.11.1-ext.2`, `composeExtended.*` in `gradle.properties`) from + # Maven Central, builds the 20-scenario executable with Liberica NIK 25, and runs it with + # JAVA_HOME and PYTHONHOME removed. notebook-e2e-native-image: name: Notebook E2E in a native image (macOS) - if: ${{ vars.COMPOSE_EXTENDED_VERSION != '' }} runs-on: macos-latest - permissions: - contents: read - packages: read env: WHEEL_DIR: ${{ github.workspace }}/.tmp/pythonx-compose-dist - COMPOSE_EXTENDED_VERSION: ${{ vars.COMPOSE_EXTENDED_VERSION }} - COMPOSE_EXTENDED_GROUP_PREFIX: ${{ vars.COMPOSE_EXTENDED_GROUP_PREFIX || 'org.thisisthepy.compose' }} - COMPOSE_EXTENDED_REPOSITORY: ${{ vars.COMPOSE_EXTENDED_REPOSITORY || 'https://maven.pkg.github.com/thisisthepy/compose-multiplatform-core-extended' }} - # GitHub Packages asks for a token even for a public package. A read:packages PAT in the secret - # GH_PACKAGES_TOKEN is used when it exists; otherwise this repository's own token. - GITHUB_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN || secrets.GITHUB_TOKEN }} - TOKEN_SOURCE: ${{ secrets.GH_PACKAGES_TOKEN != '' && 'GH_PACKAGES_TOKEN' || 'GITHUB_TOKEN' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -183,19 +172,6 @@ jobs: path: .tmp/pythonx-compose fetch-depth: 1 - # Before any build: can this token read the fork's packages? One known POM, so a refusal is - # reported as what it is rather than as a dependency resolution error minutes later. - - name: Check access to the fork's packages - run: | - group_path="${COMPOSE_EXTENDED_GROUP_PREFIX//.//}/ui" - pom="$COMPOSE_EXTENDED_REPOSITORY/$group_path/ui-desktop/$COMPOSE_EXTENDED_VERSION/ui-desktop-$COMPOSE_EXTENDED_VERSION.pom" - status=$(curl -s -o /dev/null -w '%{http_code}' -u "${GITHUB_ACTOR}:${GITHUB_TOKEN}" "$pom") - echo "reading $pom with $TOKEN_SOURCE: HTTP $status" - if [ "$status" != "200" ]; then - echo "::error::The fork's packages could not be read with $TOKEN_SOURCE (HTTP $status at $pom). Add a personal access token (classic) with read:packages as the repository secret GH_PACKAGES_TOKEN, or check COMPOSE_EXTENDED_VERSION / COMPOSE_EXTENDED_REPOSITORY." - exit 1 - fi - - name: Setup uv uses: astral-sh/setup-uv@v6 @@ -225,8 +201,4 @@ jobs: ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" - -PcomposeExtended.version="$COMPOSE_EXTENDED_VERSION" - -PcomposeExtended.groupPrefix="$COMPOSE_EXTENDED_GROUP_PREFIX" - -PcomposeExtended.mavenLocal=false - -PcomposeExtended.repository="$COMPOSE_EXTENDED_REPOSITORY" --console=plain diff --git a/docs/SPEC.md b/docs/SPEC.md index 1e0697a64..4d982f37c 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -628,7 +628,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. main-thread work to skiko's Swing queue, which starts AWT, and AWT in an image needs the JDK's `libawt`, which links `libjvm`; the fork sends it to `Dispatchers.Main` when `compose.main.dispatcher=coroutines`, and the fixture supplies a single-thread one with no AWT. - `desktopTest` keeps upstream Compose. Not on CI yet: the fork's artefacts are in mavenLocal only. + CI runs it on every push to `develop` (`desktop.yml`, job `notebook-e2e-native-image`, #251). - **B-11** (#192) The artifact walker skips every declaration whose package has an `internal` segment (`*.internal` and `*.internal.*`, as in `androidx.compose.ui.internal`), on jars and on klibs. That is the Kotlin `.api` dump convention: a library that puts a declaration in such a package has declared it diff --git a/docs/investigations/fork-desktop-paths-macos.md b/docs/investigations/fork-desktop-paths-macos.md index bbdc507c1..df39c1995 100644 --- a/docs/investigations/fork-desktop-paths-macos.md +++ b/docs/investigations/fork-desktop-paths-macos.md @@ -2,128 +2,90 @@ What python-multiplatform needs before a pythonx-compose app runs on the two desktop paths `compose-multiplatform-core-extended` and `compose-multiplatform-extended` add: a GraalVM native -image and a Kotlin/Native executable, macOS arm64 first. Read on 2026-10-17 from the fork's -`origin/extended` branches (core `61906995435`, plugin fork `fca1473a`), `~/.m2/repository`, and -this repository's `develop` (`af4debac`). Nothing here was built; every fact cites where it was read. - -## 1. What the fork has on macOS +image and a Kotlin/Native executable, macOS arm64 first. Originally read on 2026-10-17 and refreshed +on 2026-11-01 after the fork published to Maven Central (#257, #287), the notebook E2E native-image +milestone landed (#246, #249, #251), and the `#191` decisions were recorded (2026-10-25). + +## 1. What the fork publishes on Maven Central + +The fork publishes its Compose libraries, window modules and Gradle plugins to Maven Central under +`org.thisisthepy.compose`: + +- **Libraries (`1.11.1-ext.2`)**: `org.thisisthepy.compose.runtime:*`, `ui:*`, `foundation:*`, + `material:*`, `material3:*`, `animation:*`, `components:*`, `org.thisisthepy.compose.androidx.lifecycle:*`, + including `macosarm64` and `linuxx64` klibs alongside `desktop` (JVM), `android`, `ios*` and + `wasmjs`. (`material-icons-core` stays on JetBrains `1.7.3`.) +- **Window modules (`1.11.1-ext.2`)**: `org.thisisthepy.compose.window:common`, `scene`, + `graalvm-macos`, `graalvm-linux`, `native-macos`, `native-linux`. +- **Gradle plugins (`1.11.1-ext.3`)**: `org.thisisthepy.compose` and + `org.thisisthepy.kotlin.native.desktop` (`compose-gradle-plugin:1.11.1-ext.3` on Maven Central). +- **Path selection in the fork's plugin**: `compose.desktop.application { output.set(ApplicationOutput.NativeImage) }` + (with `windowing.set(ApplicationWindowing.Awt)` or `AwtFree`, and `nativeImage { ... }` registering + `packageNativeImage`, `runNativeImageAgent`, `checkNativeImageNoAwt` and `packageApplication`), or + `compose.desktop.nativeApplication { ... }` for Kotlin/Native (`createDistributableNative*`, + `runNative*`, `packageKotlinNative`). Per decision 5 on #191, consumers select the path through the + fork plugin's setting directly; `python-multiplatform` adds no duplicate path-selection DSL. | | GraalVM native image | Kotlin/Native | |---|---|---| -| Fork's claim | `extended/README.md` (core, line 12): "macOS works on both: the Kotlin/Native window (AppKit and Metal) and the GraalVM native-image window." | same line | -| Window layer | `extended/window/graalvm/graalvm-macos` (JVM lib, AppKit through `@CFunction`, `upcalls.txt` table, no reflection) | `extended/window/native/macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`) | -| Compose host | **none for this path yet**: `extended/window/scene` (JVM and `macosArm64`) hosts a Compose scene through `runComposeWindow(platform, config, surfaceFor, content)` (`scene/src/.../ComposeWindow.kt:285`), but `graalvm-macos` provides no `WindowSurface` to draw into | `scene` with `native/macos`'s `MetalSurface` | -| Measured with Compose content | `extended/native-image/hello` (plugin fork): Material 3 in an **AWT** window (`windowing = Awt`), 82 MB, "draws the same content through Skia offscreen into a PNG byte for byte the one the JVM run draws" (`extended/native-image/README.md`, 2026-10-02) | no Gradle demo in the fork; compose-rust's Amper renderer (`darkpyonix/compose-rust/renderer/macos`) consumes `native-macos` | -| AWT-free demo | `extended/native-image/hello-awtfree-link`: opens, pumps and presents an AppKit window; **draws no Compose** | none | -| Build | the plugin fork's `org.jetbrains.compose` `1.11.1-extended-dev`: `compose.desktop.application { nativeImage { graalvmHome; skikoStaticDirectory } }`, `packageNativeImage`, `runNativeImageAgent`; skiko as a static archive (`extended/skiko/build-skiko-static-jvm.sh`) | `extended/gradle-plugin-kn`: `createDistributableNative...`, `runNative...`, `packageKotlinNative` | - -**Published to mavenLocal** (JetBrains coordinates kept, as #191 says): +| Window layer | `org.thisisthepy.compose.window:graalvm-macos`, `graalvm-linux` (JVM lib, `@CFunction` + `upcalls.txt` table, no reflection) | `org.thisisthepy.compose.window:native-macos` (`macosArm64`: AppKit, Metal, `MetalSurface.kt`), `native-linux` (`linuxX64`) | +| Compose host | `org.thisisthepy.compose.window:scene` (`runComposeWindow`) | `scene` with `native-macos` / `native-linux` | +| Build | `org.thisisthepy.compose` `1.11.1-ext.3`: `compose.desktop.application { output.set(ApplicationOutput.NativeImage); nativeImage { ... } }`, `packageNativeImage`, `runNativeImageAgent` | `org.thisisthepy.kotlin.native.desktop` `1.11.1-ext.3`: `compose.desktop.nativeApplication { ... }`, `createDistributableNative*`, `runNative*` | -- `org.jetbrains.compose:ui*`, `foundation*`, `ui-text*` at `1.11.1001` (2026-10-17), including the - `macosarm64` klibs of `ui` and `foundation`. No `material3` macOS klib. -- `org.jetbrains.compose:compose-gradle-plugin:1.11.1-extended-dev`. -- `org.thisisthepy.compose.window:*:0.1.0`: `common`, `graalvm-macos`, `graalvm-linux`, `native-macos`. Not - `scene`, and not `native-linux`. -- `org.jetbrains.skiko:skiko-awt:0.144.6.1`. -- No publish script producing `1.11.1001` was found in either fork repository; the "Extended window - publish" workflow and compose-rust's `scripts/publish-window.sh` publish the window modules. - -## 2. What this repository has +## 2. What this repository has today +- **Switched to the fork's Maven Central artefacts (#257, #287).** `gradle/libs.versions.toml` uses + `org.thisisthepy.compose` `1.11.1-ext.3` (plugins) and `1.11.1-ext.2` (libraries). The artifact + walker (`ArtifactScanner.kt`) resolves `org.thisisthepy.compose.*` coordinates alongside + `org.jetbrains.compose.*`. - **Targets.** `python-multiplatform` declares `jvm("desktop")`, Android, iOS, androidNative and wasm; - no `macosArm64`, `linuxX64` or `mingwX64` (`python-multiplatform/build.gradle.kts`; the plugin's - `hostDesktopPlatform` knows macOS only as a JVM host). The sample has the same set. -- **A native image already runs Python** (`docs/platforms/graal-native-image-verification.md`, - 2026-08-17, `4687070a`): `:sample:nativeCompile` (an `Exec` of Liberica NIK 25.0.4's `native-image`, - `--no-fallback --enable-native-access=ALL-UNNAMED`) builds a headless `NativeImageMainKt`; 28 - downcalls and 5 upcalls registered, `UPCALL_OK`/`PROXY_OK`, no `libjvm`, `libpython` `dlopen`ed. - It is **headless on purpose**: the sample's `application {}` needs AWT. Not automated, not re-run. + `macosArm64` and `linuxX64` are the next interpreter targets to add (Phase A, decision 2; + `mingwX64` in Phase C). +- **A native image runs Python and stages `PYTHONHOME` beside the executable (#246, SPEC L-9).** + `pythonBindings.nativeImageDirectory` and `stageNativeImagePythonHome` copy the pruned + `python-build-standalone` prefix (`lib/python/**` and `lib/libpython.*`) into + `/python-multiplatform-home`. At run time, inside a GraalVM native image + (`org.graalvm.nativeimage.imagecode == "runtime"`), `PackagedPythonHome.resolve()` looks up + `/python-multiplatform-home` via `ProcessHandle.current().info().command()` + and hands it to CPython with `Py_SetPythonHome`. `:sample:nativeCompile` and `:sample:runNativeUpcallDemo` + stage and verify this layout with `JAVA_HOME` and `PYTHONHOME` unset (`docs/platforms/graal-native-image-verification.md`). - **The upcall table on a native image.** Desktop upcalls are FFM (`Panama.kt`), registered by the generated `META-INF/native-image/org.thisisthepy/python-multiplatform/reachability-metadata.json` - (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json`. That - metadata ships in the jar, so any native-image build that has the jar on its classpath picks it up. - (The verification note says no reflection registration is needed, while its own log shows 1,822 - types registered and the base file registers `UpcallTarget`; the note should be corrected.) -- **PYTHONHOME.** `stagePythonHome` sets `PYTHONHOME` for `JavaExec`/`Test` tasks and copies the prefix - into Compose's `prepare*AppResources`; the native demo is given `PYTHONHOME` by its run task. A - native image has no `java.home` and no packaged-resources directory, so nothing stages a prefix - **beside the executable** or finds it from there at start. -- **The notebook E2E** (`python-multiplatform-compose/fixtures/notebook-e2e`) needs no window: it - draws `PythonAppView` into an offscreen `ImageComposeScene` (`NotebookHost.kt:132`) and compares - pixels. It is a JVM `desktopTest` suite with `compose.desktop.currentOs` (Skia natives) and - `java.awt.headless=true`; no `java.awt`/`javax.swing` import. - -## 3. What each path needs from this side - -### GraalVM native image - -1. **A desktop app built by the plugin fork's `packageNativeImage`** rather than the sample's hand-made - `Exec`: the fork already solves Compose's own native-image configuration and static skiko. The - generated upcall table and FFM registrations come along in the jar's `META-INF/native-image`, so - **the upcall table carries over unchanged** (build-time table, no reflection: AGENTS.md §12.3, §13). - To check: that the fork's plugin passes `--enable-native-access=ALL-UNNAMED` (or lets an app add it) - and does not set `--no-fallback`-incompatible options that hide a missing FFM registration. -2. **PYTHONHOME beside the executable.** A task that stages the prefix and the installed wheels next to - `build/compose/native-image/main/`, and a runtime lookup (`ProcessHandle`'s executable path, a - sibling `python/` directory) when `PYTHONHOME` is unset, like `IosPythonHome` does for an app bundle. - **Staging does not carry over as is**: today's paths are a Gradle task's environment and the JVM - packaging's resources directory. -3. **The walker over the fork's jars.** Same public API, so the walk must produce the same bindings; - guarded by the stub diff of pythonx-compose #54. -4. **Selecting the path.** A Gradle plugin option (upstream Compose by default, the fork's native image - as a choice), #191's first bullet. - -### Kotlin/Native - -1. **An interpreter target**: `macosArm64` in `python-multiplatform` (cinterop against the macOS - python-build-standalone prefix the desktop JVM already downloads, a static or shipped - `libpython`), with the native `HandleTable`/trampoline path iOS and androidNative already use. - This is new work, not a switch. -2. **The klib walker** (B-2) over the fork's `macosarm64` klibs: since #234/#237 it carries handles and - interface members, but not constructors, properties or composables, and Compose's surface is mostly - composables and extensions; the K/N path needs the walker to reach about where the jar walker is. -3. **PYTHONHOME** beside the executable, as for the native image. -4. `material3` for `macosArm64`: no fork klib is published (the fork says only `ui` and `foundation` - differ from JetBrains; JetBrains' own `material3` macOS klib would have to resolve beside them). - -## 4. First milestone: the notebook E2E in a macOS GraalVM native image - -The E2E draws offscreen, so the milestone needs **no window** and none of the AWT-free pieces: - -1. A `notebook-e2e` native-image entry point (a `main` running the existing scenarios and printing a - pass/fail line per scenario, the way the fork's `hello` self-checks through `HELLO_SELF_CHECK`), - because a JUnit `desktopTest` does not run inside an image. -2. Built with the plugin fork's `packageNativeImage` (`windowing = Awt`, which is what the fork measured; - the scenes are offscreen either way), `--enable-native-access=ALL-UNNAMED`, the fork's static skiko. -3. PYTHONHOME and the pythonx-compose wheel staged beside the executable (§3, GraalVM 2). -4. Pass: the same 20 scenarios as the JVM run (pixel comparisons included), the binary run alone from - an empty directory with `JAVA_HOME` unset, and `otool -L` showing no `libjvm`. Then a CI job on a - macOS runner, as `notebook-e2e` has on the JVM (#213). - -Milestone 2 is the same app in the AWT-free window, once the fork has the pieces in §5. Kotlin/Native -follows the interpreter target (§3, K/N 1). - -## 5. Requests for the fork (through the darkpyonix lead) - -1. **Publish `org.thisisthepy.compose.window:scene`** (its `module.yaml` already enables publishing; - it is missing from mavenLocal and from the "Extended window publish" workflow's output). -2. **A `WindowSurface` for `graalvm-macos`** (Metal or GL into a Skia surface, as `native/macos`'s - `MetalSurface`), so `runComposeWindow` can host Compose content on the GraalVM path; and a demo that - draws Compose through it (`hello-awtfree-link` draws nothing today). -3. **A documented, reproducible publish of `1.11.1001`** (no script producing that version was found) - and one version line for the Compose libraries, the window modules and the Gradle plugin, so a - consumer pins one set. -4. **The native-image flags an app can add**: confirm that `packageNativeImage` lets an app pass - `--enable-native-access=ALL-UNNAMED` and merges a library's `META-INF/native-image` metadata (FFM - `foreign` registrations included). -5. **A hook to stage extra files beside the image** (a prefix directory), or confirmation that a - consumer task after `packageNativeImage` is the supported way. -6. For Kotlin/Native: whether `material3` for `macosArm64` is expected to come from JetBrains, and a - Gradle (not Amper-only) example of `scene` plus `native-macos` hosting Compose content. - -## 6. Open questions - -- Whether CPython's own native-image behaviour holds with the fork's Compose in the same image (one - process, Skia and libpython both `dlopen`ed or linked); not tried. -- Where the native-image E2E runs in CI: GitHub's macOS runners have no GraalVM by default, and the - fork's static skiko must be built or cached. + (`generateDesktopReachabilityMetadata`) plus `native-image/reachability-metadata.base.json` (which + registers `python.native.ffi.UpcallTarget`, corrected in #244). That metadata ships in the jar, so + any native-image build that has the jar on its classpath picks it up. +- **The notebook E2E runs in a macOS GraalVM native image (#249, #251, SPEC B-10).** + `python-multiplatform-compose/fixtures/notebook-e2e` builds `notebook-e2e` with `notebookE2eNativeCompile` + and runs all 20 `UI.ipynb` scenarios via `notebookE2eNativeRun` with `JAVA_HOME` and `PYTHONHOME` + removed, `compose.main.dispatcher=coroutines` (`NotebookE2eMainDispatcherFactory`, no AWT), and an + `otool -L` check forbidding `libjvm`. `.github/workflows/desktop.yml` (`notebook-e2e-native-image`) + runs this job on macOS against Maven Central. + +## 3. Remaining work by phase (per #191 decisions, 2026-10-25) + +### Phase A (in progress) + +1. **#251 (`desktop.yml`)**: `notebook-e2e-native-image` switched to Maven Central with the + `COMPOSE_EXTENDED_VERSION` guard and GitHub Packages check removed. +2. **`:sample` as a native-image output**: `:sample` sets `pythonBindings.nativeImageDirectory` so + `stageNativeImagePythonHome` stages `python-multiplatform-home/` beside `upcall-native-demo`, and + `runNativeUpcallDemo` runs the executable with `JAVA_HOME` and `PYTHONHOME` removed. +3. **Kotlin/Native desktop interpreter targets (`macosArm64`, `linuxX64`)**: add `macosArm64()` and + `linuxX64()` to `:python-multiplatform`, cinterop against the `python-build-standalone` headers + already downloaded by `downloadPython_macos_aarch64` and `downloadPython_linux_x86_64`, link and + load `libpython` dynamically beside the executable (decision 3), and wire `PackagedPythonHome` / + `PythonPayload` for desktop K/N executables. + +### Phase B + +1. **AWT-free GraalVM notebook app**: run the notebook app in an AWT-free window (`windowing = AwtFree`). +2. **Kotlin/Native macOS app**: re-check `KlibScanner` (B-2) coverage for `@Composable` functions, + constructors and properties over the fork's `macosarm64` klibs, add `macosArm64` to + `:python-multiplatform-compose`, and run a K/N macOS app. + +### Phase C + +1. **Windows (`mingwX64`)** once the fork publishes Windows artefacts. +2. **Stub diff guard (`pythonx-compose #54`)** and the final acceptance criterion: the same + `pythonx-compose` app running unchanged on upstream Compose, extended GraalVM and extended K/N + desktop builds with the notebook E2E passing in CI. diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/README.md b/python-multiplatform-compose/fixtures/notebook-e2e/README.md index af55ae435..ef0399bf7 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/README.md +++ b/python-multiplatform-compose/fixtures/notebook-e2e/README.md @@ -68,11 +68,12 @@ JAVA_HOME= ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e `site-packages/` (the wheel) and `app/`. The image finds them there with no environment. - `notebookE2eNativeRun` removes `JAVA_HOME` and `PYTHONHOME`, requires 20 passed, and fails if `otool -L` shows `libjvm`. -- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended's mavenLocal - build (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread +- The image takes `ui`, `ui-text` and `foundation` from compose-multiplatform-core-extended on Maven + Central (`composeExtended.*` in the root `gradle.properties`): upstream Compose sends its main-thread work to skiko's Swing queue, which needs AWT, and AWT needs `libjvm`. The fork sends it to `Dispatchers.Main` under `compose.main.dispatcher=coroutines`, which `NotebookE2eMainDispatcherFactory` supplies (registered by `src/desktopTest/native-image-main/` on the image's classpath only). + CI runs `notebookE2eNativeAgent` and `notebookE2eNativeRun` in `.github/workflows/desktop.yml` (`notebook-e2e-native-image`, #251). ## Scenarios diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 2e8a35dcd..2c9f5e32b 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -411,10 +411,14 @@ val nativeImageJavaLauncher = javaToolchains.launcherFor { val nativeImageOutputDir = layout.buildDirectory.dir("native/nativeCompile") +pythonBindings { + nativeImageDirectory.set(nativeImageOutputDir) +} + val nativeCompile by tasks.registering(Exec::class) { group = "native" description = "Builds upcall-native-demo with GraalVM native-image from NativeImageMain.kt" - dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar) + dependsOn(desktopMainCompilation.compileTaskProvider, pythonMultiplatformDesktopJar, "stageNativeImagePythonHome") val classpathProvider = files( desktopMainCompilation.output.allOutputs, @@ -425,7 +429,7 @@ val nativeCompile by tasks.registering(Exec::class) { val launcher = nativeImageJavaLauncher inputs.files(classpathProvider) - outputs.dir(outDir) + outputs.file(outDir.map { it.file("upcall-native-demo") }) doFirst { outDir.get().asFile.mkdirs() @@ -445,8 +449,10 @@ tasks.register("runNativeUpcallDemo") { group = "verification" description = "Runs the native-image binary built by nativeCompile and checks the upcall path" dependsOn(nativeCompile) + workingDir(nativeImageOutputDir) executable = nativeImageOutputDir.get().asFile.resolve("upcall-native-demo").absolutePath - environment("PYTHONHOME", pythonHomeForHost.get().asFile.absolutePath) + environment.remove("JAVA_HOME") + environment.remove("PYTHONHOME") // A zero exit status is not the same as a working upcall path. `PyRun_SimpleString` reports // failure for an exception raised inside the script, but a Python interpreter that never got From afec35c4f76127e652dbf36dab6e8e65459c8b04 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:26:05 +0900 Subject: [PATCH 12/61] Fix: Point Gradle toolchain resolution at GRAALVM_HOME in native-image CI (#251) --- .github/workflows/desktop.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3956921f1..cad58b5fd 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,9 +196,13 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside + # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` + # explicitly instead of picking up the runner's pre-installed Temurin 25. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" + -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain From 5b7ac9d96b8607a2283c516ee7414d19b84f2400 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:42:24 +0900 Subject: [PATCH 13/61] TypedPython: cache generated C for the pypackpack compile slot (#113) --- docs/SPEC.md | 11 +- docs/design/typedpython.md | 6 + .../src/main/python/typedpython/pipeline.py | 113 +++++++++++++----- .../src/test/python/test_generate_project.py | 111 +++++++++++++++++ 4 files changed, 209 insertions(+), 32 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_generate_project.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 4d982f37c..2e65b6f45 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -784,7 +784,16 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. the compile step. `Status: partial`: the per-module cache and its key are implemented in Python (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25, #113). + through pypackpack's compile slot (#25). + For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): + one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the + source, the dependency interface hashes and the compiler version (which covers the runtime header), + with no platform, C compiler or flag in it, because the C depends on none of them. The slot caches + C to extension itself. Tests (`test_generate_project.py`): an unchanged module is reused with a + byte-identical `.c`, a fresh cache produces the same bytes, a body change regenerates that module + only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the + emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps + building in process. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 991207854..eaf3a02d2 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -162,6 +162,12 @@ API 안정성을 확인한 뒤로 미룬다. 인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. +pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시는 두 층으로 나뉜다(#113). +`pipeline.generate_project` 는 소스 → C 만 캐시한다: 모듈마다 `.c` 하나, 그리고 `tp_runtime.h` 가 있는 +디렉터리. 키는 소스 + 의존 모듈 인터페이스 해시 + 컴파일러 버전이고, C 는 플랫폼·컴파일러·플래그를 읽지 +않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 +캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index 3f504651c..f056da353 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -154,27 +154,24 @@ class ModuleResult: interface: str = "" kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked" skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured + c_source: Path | None = None # generate_project only: the module's one `.c` (#113) @dataclass class ProjectResult: modules: list[ModuleResult] + runtime_include: Path | None = None # generate_project only: the dir holding tp_runtime.h @property def rebuilt(self) -> list[ModuleResult]: return [m for m in self.modules if m.status == "rebuilt"] -def compile_project( - sources: list[Path], - project_root: Path, - cache_dir: Path, - *, - python_include: Path | None = None, - ext_suffix: str | None = None, - runtime_dir: Path | None = None, - cost_rule: bool = True, -) -> ProjectResult: +def _project(sources: list[Path], project_root: Path, cache_dir: Path, *, cost_rule: bool, + compile_c, compiler_version: str, platform: str, flags: list[str], + c_mode: bool) -> list[ModuleResult]: + """The cached source -> artefact walk shared by compile_project (artefact: the extension) and + generate_project (artefact: the `.c`).""" sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) verified: dict[Path, list[SkippedItem]] = {} @@ -193,27 +190,12 @@ def lower(path: Path) -> ir.Module: def generate(module: ir.Module, path: Path) -> str: return cgen.generate(module, path, _relative(path, root)) if module.functions else "" - def compile_c(c_source: str, name: str, out_dir: Path) -> Path: - if not c_source: - marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) - marker.write_text("nothing compiled\n") - return marker - return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, - python_include=python_include, ext_suffix=ext_suffix) - - version = COMPILER_VERSION - if runtime_dir is not None: - version += ":" + _tree_hash(Path(runtime_dir)) if not cost_rule: - version += ":nocost" - suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") - include = cbuild._python_include(python_include) - flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] - + compiler_version += ":nocost" cache = incremental.BuildCache(cache_dir) report = cache.build_project( sources, project_root, lower=lower, generate=generate, compile_c=compile_c, - compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + compiler_version=compiler_version, platform=platform, flags=flags) side = cache_dir / "skipped" side.mkdir(exist_ok=True) @@ -230,9 +212,78 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: items = [SkippedItem(**i) for i in data["items"]] except (OSError, ValueError, KeyError, TypeError): skipped, items = {}, [] - ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + art = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + ext, c_file = (None, art) if c_mode else (art, None) kind = ("not_marked" if not frontend.is_opted_in(m.path) - else "compiled" if ext is not None else "nothing_compiled") + else "compiled" if art is not None else "nothing_compiled") out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface, - kind, items)) - return ProjectResult(out) + kind, items, c_file)) + return out + + +def _no_c_marker(name: str, out_dir: Path) -> Path: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, + cost_rule: bool = True, +) -> ProjectResult: + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags, + c_mode=False)) + + +# The platform component of a generated-C cache key. The C does not depend on the host, the target, +# the C compiler or its flags (cgen reads none of them; the target's headers and the toolchain are +# the compile slot's inputs and its own cache key, pypackpack docs/design/compile-slot.md §9), so +# one constant stands for all of them and a `.c` built on one machine is reused on another. +GENERATED_C_PLATFORM = "generated-c" + + +def generate_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + cost_rule: bool = True, +) -> ProjectResult: + """Source -> C for every module, cached (#113): the input of pypackpack's native compile slot, + which builds the extensions and caches C -> extension itself. Each compiled module gets exactly + one `.c` (`ModuleResult.c_source`, named after the module's last dotted part); a module with + nothing compiled gets none and ships as Python. `runtime_include` is the directory holding the + header-only `tp_runtime.h`, for the slot's include dirs. The key is the source, the dependency + interfaces and COMPILER_VERSION (which covers the runtime header); no toolchain flag enters it. + An unchanged module is reused with a byte-identical `.c`.""" + def write_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + return _no_c_marker(name, out_dir) + c_file = Path(out_dir) / (name.rpartition(".")[2] + ".c") + c_file.write_text(c_source) + return c_file + + return ProjectResult(_project( + sources, project_root, cache_dir, cost_rule=cost_rule, compile_c=write_c, + compiler_version=COMPILER_VERSION, platform=GENERATED_C_PLATFORM, flags=[], c_mode=True), + runtime_include=cbuild.RUNTIME_DIR) diff --git a/python-multiplatform-ksp/src/test/python/test_generate_project.py b/python-multiplatform-ksp/src/test/python/test_generate_project.py new file mode 100644 index 000000000..f5ae50a94 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_generate_project.py @@ -0,0 +1,111 @@ +"""generate_project: TypedPython caches source -> C, the compile slot builds C -> extension (#113). + +Real Pyrefly; the only C build here is the one that proves the emitted `.c` is the same program +compile_project builds. +""" +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild, frontend, pipeline + +B = decorate_compiled(textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """)) +A = decorate_compiled(textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "pkg").mkdir(parents=True) + (root / "a.py").write_text(A) + (root / "b.py").write_text(B) + (root / "pkg" / "__init__.py").write_text("") + (root / "pkg" / "plain.py").write_text(PLAIN) + return root + + +@pytest.fixture +def counts(monkeypatch): + n = {"lower": 0, "build": 0} + real_lower = frontend.lower + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + raise AssertionError("generate_project must not run the C compiler") + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def gen(project, cache): + return pipeline.generate_project( + [project / "a.py", project / "b.py", project / "pkg" / "plain.py"], project, cache) + + +def by_stem(result): + return {m.path.stem: m for m in result.modules} + + +def test_emits_one_c_per_compiled_module_and_the_runtime_dir(project, tmp_path, counts): + r = gen(project, tmp_path / "cache") + m = by_stem(r) + assert m["a"].c_source.name == "a.c" and m["b"].c_source.name == "b.c" + assert "PyInit_a" in m["a"].c_source.read_text() + assert m["a"].extension is None and m["a"].kind == "compiled" + assert m["plain"].c_source is None and m["plain"].kind == "not_marked" + assert (r.runtime_include / "tp_runtime.h").is_file() + assert counts == {"lower": 3, "build": 0} + + +def test_unchanged_module_hits_with_byte_identical_c(project, tmp_path, counts): + first = by_stem(gen(project, tmp_path / "cache")) + c_a = first["a"].c_source.read_bytes() + second = gen(project, tmp_path / "cache") + assert {m.path.stem: m.status for m in second.modules} == {"a": "reused", "b": "reused", "plain": "reused"} + assert by_stem(second)["a"].c_source.read_bytes() == c_a + assert counts["lower"] == 3 # nothing lowered again + + +def test_fresh_cache_gives_byte_identical_c(project, tmp_path): + one = by_stem(gen(project, tmp_path / "c1"))["a"].c_source.read_bytes() + two = by_stem(gen(project, tmp_path / "c2"))["a"].c_source.read_bytes() + assert one == two + + +def test_body_change_regenerates_only_that_module(project, tmp_path): + gen(project, tmp_path / "cache") + (project / "b.py").write_text(B.replace("n * 2", "n + n")) + st = {m.path.stem: m.status for m in gen(project, tmp_path / "cache").modules} + assert st == {"a": "reused", "b": "rebuilt", "plain": "reused"} + + +def test_key_ignores_toolchain(project, tmp_path, monkeypatch): + gen(project, tmp_path / "cache") + monkeypatch.setattr(cbuild, "DEFAULT_FLAGS", (*cbuild.DEFAULT_FLAGS, "-DDIFFERENT")) + monkeypatch.setattr(pipeline.sysconfig, "get_platform", lambda: "other-platform") + assert all(m.status == "reused" for m in gen(project, tmp_path / "cache").modules) + + +def test_emitted_c_is_what_compile_project_builds(project, tmp_path): + c_mod = by_stem(gen(project, tmp_path / "ccache"))["b"] + built = by_stem(pipeline.compile_project([project / "b.py"], project, tmp_path / "bcache"))["b"] + so = cbuild.build(c_mod.c_source.read_text(), "b", tmp_path / "slot") + assert cbuild.load("b", so).twice(21) == 42 + assert built.extension is not None and built.c_source is None From 831174c0a4b816d4b73445978f19812bb0c59613 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:59:00 +0900 Subject: [PATCH 14/61] Fix: Disable Gradle toolchain auto-detect in native-image CI so Liberica NIK 25 wins (#251) --- .github/workflows/desktop.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index cad58b5fd..1799d3699 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,13 +196,15 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. - # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside - # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` - # explicitly instead of picking up the runner's pre-installed Temurin 25. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache, whereas + # macOS runner images also pre-install Temurin 25 in `/Library/Java/JavaVirtualMachines`. + # Disable toolchain auto-detection and pass `org.gradle.java.installations.paths="$GRAALVM_HOME"` + # so Gradle's JavaLanguageVersion(25) launcher selects Liberica NIK 25 instead of Temurin 25. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" + -Porg.gradle.java.installations.auto-detect=false -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain From 693252cf0764dbf5ab07983059f5e55248a9b84e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:10:49 +0900 Subject: [PATCH 15/61] Fix: Require BellSoft vendor for JDK 25 native-image toolchain (#251) --- .github/workflows/desktop.yml | 8 +++----- .../fixtures/notebook-e2e/build.gradle.kts | 5 ++++- sample/build.gradle.kts | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 1799d3699..6a80f31e0 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -196,15 +196,13 @@ jobs: # Records Compose and skiko's configuration against the fork version this run uses, then builds # the image and runs it with JAVA_HOME and PYTHONHOME removed: 20 of 20 and no libjvm. - # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache, whereas - # macOS runner images also pre-install Temurin 25 in `/Library/Java/JavaVirtualMachines`. - # Disable toolchain auto-detection and pass `org.gradle.java.installations.paths="$GRAALVM_HOME"` - # so Gradle's JavaLanguageVersion(25) launcher selects Liberica NIK 25 instead of Temurin 25. + # `setup-graalvm` installs Liberica NIK under `GRAALVM_HOME` in the runner toolcache outside + # `/Users/runner/hostedtoolcache/Java_*`, so pass `org.gradle.java.installations.paths` + # explicitly so Gradle's BellSoft JDK 25 toolchain resolution finds it. - name: Build and run the native image run: > ./gradlew :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeAgent :python-multiplatform-compose:fixtures:notebook-e2e:notebookE2eNativeRun -PpythonxComposeWheel="$WHEEL_DIR" - -Porg.gradle.java.installations.auto-detect=false -Porg.gradle.java.installations.paths="$GRAALVM_HOME" --console=plain diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts index 8c4915a86..0c269b73a 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts @@ -280,7 +280,10 @@ val e2eMainClass = "fixture.notebook.NotebookE2eMainKt" val e2eNativeConfig = layout.projectDirectory.dir("src/desktopTest/native-image") val e2eNativeDir = layout.buildDirectory.dir("native/notebook-e2e") val e2eAppDir = layout.projectDirectory.dir("src/desktopTest/python") -val e2eJdk25 = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) } +val e2eJdk25 = javaToolchains.launcherFor { + languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) +} fun JavaExec.notebookE2eOnTheJvm() { dependsOn(e2eTestCompilation.compileTaskProvider, installPythonxComposeWheel) diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 2c9f5e32b..326e10cb7 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -407,6 +407,7 @@ tasks.matching { it.name == "run" }.configureEach { val nativeImageJavaLauncher = javaToolchains.launcherFor { languageVersion.set(JavaLanguageVersion.of(25)) + vendor.set(JvmVendorSpec.BELLSOFT) } val nativeImageOutputDir = layout.buildDirectory.dir("native/nativeCompile") From 986c736a5003c78024a5e8013b9c817b612b740b Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:38:31 +0900 Subject: [PATCH 16/61] TypedPython: narrow GCC fastmath to reassociation and FMA contract (#368) --- docs/SPEC.md | 6 ++ .../src/main/python/typedpython/cgen.py | 6 +- .../src/test/python/test_pipeline.py | 83 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 2e65b6f45..14c5ece6f 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -748,6 +748,12 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. caller without fastmath): **this is the one path in TypedPython that is explicitly NOT CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round differently from the interpreter so float reductions and FMA loops can vectorise). + The three compilers allow the same scope (#368): reassociation and FMA contraction only. Clang uses + `#pragma clang fp reassociate(on) contract(fast)`, MSVC `float_control(precise, off)` with + `fp_contract(on)`, GCC `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` + (the two extra options are what associative-math needs to take effect). None of them assumes finite + values, so a NaN check (`x != x`, `isnan`) in a fastmath function still works; GCC's `fast-math` is + not used (`test_pipeline.py`: the macro test, and the NaN-check test run under a real GCC on Linux CI). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 940837ed9..295484182 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -758,7 +758,11 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") w("#elif defined(__GNUC__)") - w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(\\\"fast-math\\\")\")") + # The same scope as clang and MSVC (#368): reassociation and FMA contraction only. + # associative-math takes effect only with no-signed-zeros and no-trapping-math; never + # fast-math, whose finite-math-only folds `x != x` and isnan away. + w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(" + "\\\"associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast\\\")\")") w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") w("#elif defined(_MSC_VER)") w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index 18355aec4..dcfb7d6da 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -238,3 +238,86 @@ def calls_fast_and_adds(a: float, b: float, c: float) -> float: got_fast = mod.fma_fast(a, a, c) assert isinstance(got_fast, float) + + +_NAN_SRC = """ +@compiled(fastmath=True) +def nan_count(xs: list[float]) -> int: + n = 0 + for i in range(len(xs)): + x = xs[i] + if x != x: + n += 1 + return n + +@compiled(fastmath=True) +def is_nan(x: float) -> bool: + return x != x +""" + + +def _real_gcc(): + """A GCC driver that is really GCC (macOS `gcc` is clang), or None.""" + import shutil + import subprocess + for name in ("gcc", "gcc-15", "gcc-14", "gcc-13"): + exe = shutil.which(name) + if exe: + out = subprocess.run([exe, "--version"], capture_output=True, text=True).stdout + if "clang" not in out.lower(): + return exe + return None + + +def test_gcc_fastmath_scope_is_reassociation_and_contract_only(write): + """#368: GCC's fastmath macro allows the same scope as clang and MSVC, reassociation and FMA + contraction (plus the no-signed-zeros / no-trapping-math that associative-math needs); never + `fast-math`, whose finite-math-only folds `x != x` to false.""" + from typedpython import cgen, frontend, verify + + path = write("fm_scope.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + c_text = cgen.generate(proved, path) + gcc_line = next(l for l in c_text.splitlines() if "define TP_FASTMATH_BEGIN" in l and "GCC" in l) + assert "fast-math" not in gcc_line + for opt in ("associative-math", "no-signed-zeros", "no-trapping-math", "fp-contract=fast"): + assert opt in gcc_line, (opt, gcc_line) + assert "finite-math" not in gcc_line + + +def test_fastmath_keeps_nan_checks(write, tmp_path): + """#368: `x != x` inside a fastmath function still detects NaN (host compiler).""" + path = write("fm_nan.py", _NAN_SRC) + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True and mod.is_nan(1.0) is False + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_fastmath_keeps_nan_checks_under_gcc(write, tmp_path, monkeypatch): + """#368: the same under a real GCC (Linux CI); skipped where no GCC is installed.""" + import sys + from typedpython import cbuild + gcc = _real_gcc() + if gcc is None or sys.platform == "win32": + pytest.skip("no real GCC on this host") + monkeypatch.setattr(cbuild, "compiler", lambda: [gcc]) + path = write("fm_nan_gcc.py", _NAN_SRC) + # GCC only warns on an optimize option it does not know; make that an error so a dropped + # option cannot pass silently. + from typedpython import cgen, frontend, verify + proved, _ = verify.verify(frontend.lower(path)) + c_file = tmp_path / "fm_nan_gcc.c" + c_file.write_text(cgen.generate(proved, path)) + chk = subprocess.run([gcc, "-fsyntax-only", "-Werror=pragmas", "-Werror=attributes", + f"-I{cbuild._python_include(None)}", f"-I{cbuild.RUNTIME_DIR}", str(c_file)], + capture_output=True, text=True) + assert chk.returncode == 0, chk.stderr + res = pipeline.compile_module(path, tmp_path / "out", cost_rule=False) + assert {f.name for f in res.module.functions} == {"nan_count", "is_nan"}, res.skipped + mod = pipeline.load(res) + nan = float("nan") + assert mod.is_nan(nan) is True + assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 From decc90e735808f2e3ae49e33db57f814ba008e49 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:54:20 +0900 Subject: [PATCH 17/61] Feat: add macosArm64 and linuxX64 Kotlin/Native desktop targets (#191) --- .github/workflows/desktop.yml | 25 +++++ docs/SPEC.md | 3 +- python-multiplatform/build.gradle.kts | 58 +++++++++- .../env/PackagedPythonHome.desktopNative.kt | 100 ++++++++++++++++++ .../env/PythonPayload.desktopNative.kt | 24 +++++ .../env/DesktopNativePythonHomeTest.kt | 93 ++++++++++++++++ .../python/multiplatform/Platform.linuxX64.kt | 39 +++++++ .../python/multiplatform/ThreadId.linuxX64.kt | 9 ++ .../env/ExecutableDir.linuxX64.kt | 21 ++++ .../multiplatform/Platform.macosArm64.kt | 49 +++++++++ .../multiplatform/ThreadId.macosArm64.kt | 8 ++ .../env/ExecutableDir.macosArm64.kt | 27 +++++ 12 files changed, 454 insertions(+), 2 deletions(-) create mode 100644 python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt create mode 100644 python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt create mode 100644 python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt create mode 100644 python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt create mode 100644 python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 6a80f31e0..7d6003ba4 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -51,6 +51,31 @@ jobs: # Using --console=plain so logs are readable in GitHub Actions run: ./gradlew :python-multiplatform:desktopTest --console=plain + desktop-native-linux: + name: Desktop Kotlin/Native Tests (Linux x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + cache: 'gradle' + + - name: Cache Konan + uses: actions/cache@v4 + with: + path: ~/.konan + key: ${{ runner.os }}-konan-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-konan- + + - name: Run Linux x86_64 Kotlin/Native Tests + run: ./gradlew :python-multiplatform:linuxX64Test --rerun --console=plain + # The dev agent's protocol on the other desktop OSes (#238). Its socket options differ per platform # (SO_EXCLUSIVEADDRUSE on Windows, SO_REUSEADDR on POSIX), and the port-takeover and restart tests are # what check them. Stdlib only: no JDK, no Gradle. diff --git a/docs/SPEC.md b/docs/SPEC.md index 2e65b6f45..13e3a29f4 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -26,12 +26,13 @@ fail, then implement. | Platform | Kotlin target(s) | Downcall mechanism | Upcall entry | Test path | |---|---|---|---|---| | Desktop JVM (macOS; Linux/Windows wired, unverified) | `jvm("desktop")` | Panama FFM, `invokeExact` | FFM upcall stub | `desktopTest → jvmTest → commonTest` | +| Desktop native (macOS Arm64, Linux x86_64) | `macosArm64`, `linuxX64` | cinterop | `@CName` symbol | `{macosArm64Test, linuxX64Test} → desktopNativeTest → nativeTest → commonTest` | | Android (ART) | `androidTarget` | JNI via `RegisterNatives` | JNI upcall | `androidInstrumentedTest → jvmTest → commonTest` (device) | | iOS | `iosArm64`, `iosSimulatorArm64`, `iosX64` | cinterop | `@CName` symbol | `iosSimulatorArm64Test → nativeTest → commonTest` | | Android native | `androidNativeArm64`, `androidNativeX64` | cinterop | `@CName` symbol | `androidNativeArm64Test → artTest → nativeTest → commonTest` (device, via adb) | | Web (experimental) | `wasmJs` (Node, browser) | JS / Emscripten CPython | `@WasmExport` | `wasmJsTest` (Node suite; browser subset) | -- **P-1** The library compiles for every target above. `Status: implemented` for desktop, Android, +- **P-1** The library compiles for every target above. `Status: implemented` for desktop, `macosArm64`, `linuxX64`, Android, iosSimulatorArm64, androidNativeArm64 and wasmJs (each has a test source set). `partial` for `iosArm64`, `iosX64`, `androidNativeX64` (compile-wired, no tests run there). - **P-2** Linux and Windows desktop: the build downloads and links the right CPython archive. diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index 46e46c00b..bb3ac4824 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -89,6 +89,7 @@ buildkonfig { } val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".") +val abiSuffix = if (pythonFreeThreaded) "t" else "" /** * The `ftp/python//` directory holding a release, which drops any pre-release suffix: @@ -1245,7 +1246,8 @@ kotlin { */ listOf( iosArm64(), iosSimulatorArm64(), iosX64(), - androidNativeArm64(), androidNativeX64() + androidNativeArm64(), androidNativeX64(), + macosArm64(), linuxX64() ).forEach { nativeTarget -> nativeTarget.apply { val targetABI = when(konanTarget) { @@ -1254,16 +1256,20 @@ kotlin { IOS_ARM64 -> "ios-arm64" IOS_X64 -> "ios-arm64_x86_64-simulator" IOS_SIMULATOR_ARM64 -> "ios-arm64_x86_64-simulator" + MACOS_ARM64 -> "macos-aarch64" + LINUX_X64 -> "linux-x86_64" else -> throw RuntimeException("Unsupported ABI: $konanTarget") } val targetExtractDir = when(konanTarget.family) { Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}/prefix" Family.IOS -> "$extractedDir/ios/Python.xcframework" + Family.OSX, Family.LINUX -> "$extractedDir/$targetABI$desktopFlavourSuffix/python" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } val targetIncludePath = when(konanTarget.family) { Family.ANDROID -> "$targetExtractDir/include/python$libVersion" Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers" + Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$abiSuffix" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } @@ -1331,6 +1337,28 @@ kotlin { "-framework", "Python", "-F$targetExtractDir/$targetABI", "-rpath", "$targetExtractDir/$targetABI" )) + } else if (konanTarget.family == Family.OSX) { + getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( + "-L$targetExtractDir/lib", + "-lpython$libVersion$abiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + } else if (konanTarget.family == Family.LINUX) { + // python-build-standalone ships libpythonX.Y.so as a symlink to libpythonX.Y.so.1.0, + // which Gradle's tarTree extracts as a 0-byte file; stageDesktopLibraries resolves + // it into a real libpythonX.Y.so under build/desktop-libs/lib/linux-x86_64. + val stagedLinuxLibDir = layout.buildDirectory.dir("desktop-libs/lib/linux-x86_64").get().asFile.absolutePath + getTest(NativeBuildType.DEBUG).apply { + linkerOpts.addAll(listOf( + "-L$stagedLinuxLibDir", + "-L$targetExtractDir/lib", + "-lpython$libVersion$abiSuffix", + "-Wl,-rpath,$targetExtractDir/lib" + )) + linkTaskProvider.configure { + dependsOn("stageDesktopLibraries") + } + } } } } @@ -1426,6 +1454,15 @@ kotlin { androidNativeX64Main.dependsOn(artMain) androidNativeArm64Main.dependsOn(artMain) + val desktopNativeMain by creating { + kotlin.srcDir("src/desktopNativeMain/kotlin") + } + val macosArm64Main by getting + val linuxX64Main by getting + desktopNativeMain.dependsOn(nativeMain) + macosArm64Main.dependsOn(desktopNativeMain) + linuxX64Main.dependsOn(desktopNativeMain) + // `src/nativeTest` existed as a directory long before this line, and nothing pointed at // it -- so its `GCLeakTest.native.kt` was dead source, and the same file had been copied // byte-for-byte into all five target test source sets to compensate. This is the mirror of @@ -1439,6 +1476,13 @@ kotlin { "androidNativeX64Test", "androidNativeArm64Test", ).forEach { getByName(it).dependsOn(nativeTest) } + val desktopNativeTest by creating { + kotlin.srcDir("src/desktopNativeTest/kotlin") + } + desktopNativeTest.dependsOn(nativeTest) + listOf("macosArm64Test", "linuxX64Test") + .forEach { getByName(it).dependsOn(desktopNativeTest) } + // ...and `artTest` completes the mirror for the other half: things true of androidNative // but not of iOS. Without it such a test has nowhere to live except duplicated into both // androidNative target source sets, which is the arrangement the comment above describes @@ -1452,6 +1496,18 @@ kotlin { } } +tasks.named("macosArm64Test") { + dependsOn("downloadPython_macos_aarch64") + val pythonHome = file("$extractedDir/macos-aarch64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + +tasks.named("linuxX64Test") { + dependsOn("downloadPython_linux_x86_64") + val pythonHome = file("$extractedDir/linux-x86_64$desktopFlavourSuffix/python").absolutePath + environment("PYTHONHOME", pythonHome) +} + /** * The iOS `Python.framework` ships only the interpreter binary and headers, it carries no * standard library. `Py_Initialize()` therefore aborts the process with diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt new file mode 100644 index 000000000..a338dc59e --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt @@ -0,0 +1,100 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import platform.posix.S_IFDIR +import platform.posix.S_IFMT +import platform.posix.stat + +/** + * The directory containing the running executable, or null when the OS cannot report it. + */ +internal expect fun executableDirectory(): String? + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) finds its packaged CPython + * prefix (SPEC L-9 / P-1, #191). + * + * A desktop native binary has no JVM, no system properties and no launcher to set `PYTHONHOME`: + * like a GraalVM native image (#246), its prefix ships beside the executable under + * [DIRECTORY_NAME] (or, inside a macOS `.app` bundle, under `../Resources/`[DIRECTORY_NAME]). + */ +internal object DesktopNativePythonHome { + + /** Matches `PackagedPythonHome.DIRECTORY_NAME` (`jvmMain`) and `IosPythonHome.DIRECTORY_NAME` (`iosMain`). */ + const val DIRECTORY_NAME = "python-multiplatform-home" + + /** A resolved prefix and a human-readable account of where it came from, for error messages. */ + data class Resolved(val path: String, val source: String) + + /** [resolve] against this process's environment and executable directory. */ + fun resolve(): Resolved? = resolve( + environmentHome = readEnvVar("PYTHONHOME"), + executableDir = executableDirectory(), + isDirectory = ::isDirectory, + ) + + /** + * The packaged prefix, or null when there is none to apply. + * + * 1. A non-blank `PYTHONHOME` in the environment wins, and the answer is null: CPython reads it + * itself and [PythonHomeCheck] already checked it. This keeps the Gradle host test tasks + * (`macosArm64Test`, `linuxX64Test`) and an explicit caller environment working unchanged. + * 2. `/`[DIRECTORY_NAME], when that directory exists. + * 3. `/../Resources/`[DIRECTORY_NAME], when that directory exists (a macOS + * `.app` bundle places the binary in `Contents/MacOS/` and resources in `Contents/Resources/`). + */ + fun resolve( + environmentHome: String?, + executableDir: String?, + isDirectory: (String) -> Boolean, + ): Resolved? { + if (!environmentHome.isNullOrBlank()) return null + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return null + val beside = "$dir/$DIRECTORY_NAME" + if (isDirectory(beside)) { + return Resolved(beside, "$DIRECTORY_NAME beside the executable ($dir)") + } + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/$DIRECTORY_NAME" + if (isDirectory(resources)) { + return Resolved(resources, "$DIRECTORY_NAME in the app bundle's Resources directory ($parent/Resources)") + } + } + return null + } + + /** `stat(2)`'s answer: a directory exists at [path] (a regular file there is not one). */ + @OptIn(ExperimentalForeignApi::class) + internal fun isDirectory(path: String): Boolean = memScoped { + val info = alloc() + if (stat(path, info.ptr) != 0) return false + (info.st_mode.toInt() and S_IFMT) == S_IFDIR + } +} + +/** + * Hands a packaged prefix to CPython with `Py_SetPythonHome` before `Py_Initialize()`. + * + * `Py_DecodeLocale` is documented by CPython as safe to call before initialisation and produces + * storage that lives for the duration of the process, as `Py_SetPythonHome` requires. + */ +@OptIn(ExperimentalForeignApi::class) +@Suppress("DEPRECATION") +internal actual fun applyPackagedPythonHome() { + val home = DesktopNativePythonHome.resolve() ?: return + try { + PythonHomeCheck.verifyOrThrow(home.path) + } catch (e: IllegalStateException) { + throw IllegalStateException( + "The Python home this application was packaged with (${home.source}) is unusable: ${e.message}", + e, + ) + } + val wide = python.native.ffi.bindings.Py_DecodeLocale(home.path, null) + ?: throw IllegalStateException("Py_DecodeLocale could not decode the packaged Python home '${home.path}'.") + python.native.ffi.bindings.Py_SetPythonHome(wide) +} diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt new file mode 100644 index 000000000..cb3c3d1e0 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PythonPayload.desktopNative.kt @@ -0,0 +1,24 @@ +package python.multiplatform.env + +/** + * Desktop Kotlin/Native's answer to "where did the payload land": `python/` beside the executable + * or, in a macOS `.app` bundle, under `../Resources/python`. + */ +internal actual fun discoverStagedPayloadRoots(): List = + discoverStagedPayloadRoots(executableDirectory(), ::pathIsAccessible) + +internal fun discoverStagedPayloadRoots( + executableDir: String?, + isAccessible: (String) -> Boolean, +): List { + val dir = executableDir?.takeIf { it.isNotBlank() }?.trimEnd('/') ?: return emptyList() + val roots = mutableListOf() + val beside = "$dir/${PythonPayload.PAYLOAD_ROOT}" + if (isAccessible(beside)) roots += beside + val parent = dir.substringBeforeLast('/', missingDelimiterValue = "") + if (parent.isNotEmpty()) { + val resources = "$parent/Resources/${PythonPayload.PAYLOAD_ROOT}" + if (resources != beside && isAccessible(resources)) roots += resources + } + return roots +} diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt new file mode 100644 index 000000000..28fb45434 --- /dev/null +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/env/DesktopNativePythonHomeTest.kt @@ -0,0 +1,93 @@ +package python.multiplatform.env + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Where a desktop Kotlin/Native application (`macosArm64`, `linuxX64`) resolves its packaged + * CPython prefix and payload from. + */ +class DesktopNativePythonHomeTest { + + private val besideBinary = "/apps/demo/python-multiplatform-home" + private val inAppBundle = "/apps/Demo.app/Contents/Resources/python-multiplatform-home" + + @Test + fun theDirectoryNameMatchesThePluginAndDesktopAndIos() { + assertEquals("python-multiplatform-home", DesktopNativePythonHome.DIRECTORY_NAME) + } + + @Test + fun anEnvironmentPythonHomeWinsAndNothingIsApplied() { + assertNull( + DesktopNativePythonHome.resolve( + environmentHome = "/from/env", + executableDir = "/apps/demo", + isDirectory = { true }, + ) + ) + } + + @Test + fun thePrefixBesideTheExecutableIsUsedWhenPresent() { + for (environment in listOf(null, "", " ")) { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = environment, + executableDir = "/apps/demo", + isDirectory = { it == besideBinary }, + ) + assertEquals(besideBinary, resolved?.path, "PYTHONHOME='$environment'") + assertTrue("beside the executable" in resolved!!.source, resolved.source) + } + } + + @Test + fun anAppBundleResourceDirectoryIsCheckedWhenNothingSitsBesideTheBinary() { + val resolved = DesktopNativePythonHome.resolve( + environmentHome = null, + executableDir = "/apps/Demo.app/Contents/MacOS", + isDirectory = { it == inAppBundle }, + ) + assertEquals(inAppBundle, resolved?.path) + assertTrue("Resources" in resolved!!.source, resolved.source) + } + + @Test + fun anUnpackagedExecutableBehavesAsBefore() { + assertNull(DesktopNativePythonHome.resolve(null, "/apps/demo") { false }) + assertNull(DesktopNativePythonHome.resolve(null, null) { true }) + assertNull(DesktopNativePythonHome.resolve(null, "") { true }) + } + + @Test + fun theHostTestTaskPassesPythonHomeAndExecutableDirectoryResolves() { + val home = assertNotNull(readEnvVar("PYTHONHOME"), "the host test task must pass PYTHONHOME") + assertNull(DesktopNativePythonHome.resolve()) + val exeDir = assertNotNull(executableDirectory(), "executableDirectory() returned null") + assertTrue(DesktopNativePythonHome.isDirectory(exeDir), exeDir) + assertTrue(DesktopNativePythonHome.isDirectory(home), home) + assertFalse(DesktopNativePythonHome.isDirectory("$home/does-not-exist")) + } + + @Test + fun payloadDiscoveryFindsBesideExecutableAndBundleResources() { + assertEquals( + listOf("/apps/demo/python"), + discoverStagedPayloadRoots("/apps/demo") { it == "/apps/demo/python" }, + ) + assertEquals( + listOf("/apps/Demo.app/Contents/Resources/python"), + discoverStagedPayloadRoots("/apps/Demo.app/Contents/MacOS") { + it == "/apps/Demo.app/Contents/Resources/python" + }, + ) + assertEquals( + emptyList(), + discoverStagedPayloadRoots("/apps/demo") { false }, + ) + } +} diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt new file mode 100644 index 000000000..dfa1251af --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/Platform.linuxX64.kt @@ -0,0 +1,39 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import platform.posix.uname +import platform.posix.utsname + +/** + * The Linux Kotlin/Native (`linuxX64`) view of the running host. + */ +object LinuxNativePlatform : Platform { + override val os = OSType.Linux + override val arch: String = machineName() + override val version: String = kernelRelease() + override val versionCode: Int? = null + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun machineName(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.machine.toKString().ifEmpty { "unknown" } +} + +@OptIn(ExperimentalForeignApi::class) +private fun kernelRelease(): String = memScoped { + val info = alloc() + if (uname(info.ptr) != 0) return "unknown" + info.release.toKString().ifEmpty { "unknown" } +} + +actual val currentPlatform: Platform = LinuxNativePlatform diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt new file mode 100644 index 000000000..94cbe50da --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/ThreadId.linuxX64.kt @@ -0,0 +1,9 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.convert + +/** glibc's `pthread_t` is a plain unsigned integral typedef (`unsigned long`), not a pointer. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self().convert() diff --git a/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt new file mode 100644 index 000000000..72e10a9ea --- /dev/null +++ b/python-multiplatform/src/linuxX64Main/kotlin/python/multiplatform/env/ExecutableDir.linuxX64.kt @@ -0,0 +1,21 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ByteVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.allocArray +import kotlinx.cinterop.convert +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.set +import kotlinx.cinterop.toKString +import platform.posix.PATH_MAX +import platform.posix.readlink + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? = memScoped { + val buf = allocArray(PATH_MAX) + val len = readlink("/proc/self/exe", buf, (PATH_MAX - 1).convert()) + if (len <= 0) return null + buf[len.toInt()] = 0 + val path = buf.toKString() + path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt new file mode 100644 index 000000000..6be1e9ab1 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/Platform.macosArm64.kt @@ -0,0 +1,49 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.toKString +import kotlinx.cinterop.useContents +import platform.Foundation.NSProcessInfo +import platform.posix.uname +import platform.posix.utsname + +/** + * The macOS Kotlin/Native (`macosArm64`) view of the running host. + */ +object MacosNativePlatform : Platform { + override val os = OSType.MacOS + override val arch: String = getDeviceArch() + override val version: String = getOSVersionString() + override val versionCode: Int = getOSVersionCode() + override val versionText = "SDK $versionCode" + override val platformType = PlatformType.Native + override val platformVersion: String? = null + + override fun toString(): String = name +} + +@OptIn(ExperimentalForeignApi::class) +private fun getDeviceArch(): String = memScoped { + val systemInfo = alloc() + if (uname(systemInfo.ptr) != 0) { + throw RuntimeException("Failed to get device arch info") + } + systemInfo.machine.toKString() +} + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionString(): String = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + "$majorVersion.$minorVersion.$patchVersion" + } + +@OptIn(ExperimentalForeignApi::class) +private fun getOSVersionCode(): Int = + NSProcessInfo.processInfo.operatingSystemVersion().useContents { + majorVersion.toInt() * 10_000 + minorVersion.toInt() * 100 + patchVersion.toInt() + } + +actual val currentPlatform: Platform = MacosNativePlatform diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt new file mode 100644 index 000000000..d19062749 --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/ThreadId.macosArm64.kt @@ -0,0 +1,8 @@ +package python.multiplatform + +import kotlinx.cinterop.ExperimentalForeignApi + +/** Darwin's `pthread_self()` returns a pointer; its bit pattern is the thread identity. */ +@OptIn(ExperimentalForeignApi::class) +internal actual fun currentThreadId(): Long = + platform.posix.pthread_self()?.rawValue?.toLong() ?: 0L diff --git a/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt new file mode 100644 index 000000000..38fb7176b --- /dev/null +++ b/python-multiplatform/src/macosArm64Main/kotlin/python/multiplatform/env/ExecutableDir.macosArm64.kt @@ -0,0 +1,27 @@ +package python.multiplatform.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.free +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.toKString +import platform.Foundation.NSBundle +import platform.Foundation.NSProcessInfo +import platform.posix.realpath + +@OptIn(ExperimentalForeignApi::class) +internal actual fun executableDirectory(): String? { + val raw = NSBundle.mainBundle.executablePath + ?: (NSProcessInfo.processInfo.arguments.firstOrNull() as? String) + ?: return null + val resolved = realpath(raw, null) + val path = if (resolved != null) { + try { + resolved.toKString() + } finally { + nativeHeap.free(resolved) + } + } else { + raw + } + return path.substringBeforeLast('/', missingDelimiterValue = "").takeIf { it.isNotEmpty() } +} From 5d93fe927e21ce3648a5ecc3954542d0e83f4890 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:21:07 +0900 Subject: [PATCH 18/61] TypedPython test: ignore a probe that succeeds only after the call returned The macOS run of develop c50b53d failed in test_other_thread_release_and_append_get_bufferror_during_a_memoryview_call with 113 BufferErrors followed by one success: the compiled call returned between the worker's done check and its next probe. A success now counts only when the call keeps running for 0.05 s after it. --- .../src/test/python/test_buffer_memoryview.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index 8a616abc8..e622536b9 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -498,7 +498,12 @@ def worker(): except Exception as e: # noqa: BLE001 seen[name][1].append(repr(e)) else: - seen[name][1].append("succeeded") + # The call can return between the `done` check above and this probe; + # the buffer is then legitimately free. A probe that succeeds while the + # buffer is really unheld does so ~0.05 s into a ~0.6 s call, so `done` + # stays unset for far longer than this grace period. + if not done.wait(0.05): + seen[name][1].append("succeeded") return time.sleep(0.001) From 8ccfda4ba2be9c66024fbb8bf17014ae1635aca6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:39:36 +0900 Subject: [PATCH 19/61] TypedPython: fill a fresh New's slots by direct release stores (#125) tp_new_fixed no longer goes through PyMember_SetOne for each slot of the object it has just allocated. The store is the one SetOne makes for a Py_T_OBJECT_EX slot (release store of a new reference) without the per-object critical section and the release of the old value, which is NULL after tp_alloc. Nothing between tp_alloc and the last store reaches a safepoint, so no stop-the-world can hand the object to another thread before it is filled. The design note also records why a freelist and later GC tracking were not done. --- docs/design/typedpython.md | 28 +++++++++++++++++++ .../main/python/typedpython/runtime/API.md | 3 +- .../python/typedpython/runtime/tp_runtime.h | 22 +++++++++++++-- .../src/test/python/test_cgen_classes.py | 14 ++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index eaf3a02d2..31e8a0f64 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -221,6 +221,34 @@ GC 추적), refcount 증감, 해제(`subtype_dealloc`)를 그대로 치르기 더 가려면 객체를 아예 만들지 않거나(가상 객체), 증감을 빼거나(refcount 생략), 한꺼번에 할당·해제해야 한다(영역 할당). 셋 다 `id()`, `__del__` 시점, `gc` 관측과 부딪칠 수 있다. +**실제 객체(N-11)의 생성 비용 (#125, 2026-11-05).** 프로파일(`benchmarks/typedpython/results/2026-10-25-binary-trees-alloc-profile.md`) +가운데 관측을 바꾸지 않고 줄일 수 있는 것은 다음과 같이 처리했다. +- 호출마다 스레드 상태를 한 번만 읽는다(#273). +- `tp_new_fixed` 는 새 객체의 슬롯을 `PyMember_SetOne` 대신 release store 로 직접 채운다. `SetOne` 이 + `Py_T_OBJECT_EX` 슬롯에 하는 일(객체 critical section 안에서 `Py_XNewRef(v)` 의 release store, 그 뒤 옛 값 + `Py_XDECREF`)에서 critical section 과 옛 값 해제만 뺀 것이다. 옛 값은 `tp_alloc` 이 0 으로 채웠으므로 NULL 이다. + critical section 을 빼도 되는 근거: `tp_alloc` 부터 마지막 슬롯 저장까지 스레드 상태를 놓는 호출도, eval + breaker 검사도 없어서 safepoint 가 없다. 그래서 stop-the-world 가 필요한 `gc.get_objects()` / + `gc.get_referrers()` 는 채우기가 끝나기 전에 이 객체를 다른 스레드에 줄 수 없다. 오히려 기존 critical + section 은 경합 시 스레드 상태를 놓을 수 있어서, 반쯤 채운 객체가 보이는 실행이 가능했다(그것도 인터프리터의 + `__init__` 실행과 같은 상태라 유효했다). 슬롯의 종류·쓰기 가능·오프셋 범위는 `tp_class_capture` 가 init 에서 + 확인한다. 참조 수: 슬롯은 `Py_NewRef` 로 자기 참조를 갖는다(이것을 빼는 변이는 `test_cgen_classes.py` 에서 + segfault 로 잡힌다). 측정(참고용, load 24~27): binary-trees `check(make(18))` 의 CPU 시간 중앙값이 develop + 대비 1.08배 빠르다. 같은 부하에서 벽시계 시간은 한 회차 안에서도 2~22초로 흔들려 쓸 수 없었다. + +하지 않은 것과 이유: +- **타입별 freelist.** 해제된 객체의 메모리를 붙잡으려면 클래스의 `tp_dealloc` 를 바꿔야 한다. 이것은 + 2026-10-25 결정(#125 댓글)에서 미뤘다: 클래스가 뒤에 `__del__`, `__weakref__`, 서브클래스를 얻으면 바뀐 dealloc 이 + 관측된다. free-threaded 빌드에서는 해제된 GC 객체의 메모리를 다른 스레드가 lock-free 로 읽을 수 있어서 + (`_Py_TryIncref` 계열) CPython 은 mimalloc 의 지연 해제(QSBR)에 기댄다. 확장이 따로 붙잡는 freelist 는 이 보장 + 밖에 있으므로, 이 결정을 다시 열 때 그 근거도 함께 써야 한다. +- **GC 추적을 늦추기/생략.** 아직 사이클을 만들 수 없는 노드(자식이 `None`)를 추적하지 않는 방식은 + `gc.is_tracked`, `gc.get_objects()` 로 관측된다. 그리고 인터프리터가 나중에 슬롯에 자기 자신을 넣으면 + (`PyMember_SetOne` 은 추적을 다시 켜지 않는다) 사이클이 수거되지 않는다. 컬렉션 횟수를 줄이는 방식(할당 계수 + 묶기, 임계값 조정)은 컬렉션 시점을 바꾼다. 둘 다 SPEC N-12 의 "GC 시점" 판단에 속하므로 오너 결정 사항이다. +- binary-trees 의 `main` 은 정책(#150)으로 컴파일되지 않고 순수하지도 않아서(`print`, `sys.argv`), 실제 객체를 + 아예 없애는 N-12b 영역이 이 벤치마크에는 적용되지 않는다. + **동일성의 정의.** 컴파일된 코드의 모든 실행은 **어떤 유효한 CPython 실행과 구별할 수 없어야 한다.** 구별은 관측으로만 할 수 있으므로, 설계는 "관측 지점" 목록과 "관측 지점에 닿기 전에 실제 객체로 되돌린다"는 규칙으로 이루어진다. CPython 자체가 정하지 않는 것(스레드 전환 시점)은 CPython 이 고를 수 있는 실행 중 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 9c30cdad8..4d0e4f25f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -389,7 +389,8 @@ Slot access and the slow paths: int tp_field_set(PyObject *obj, PyMemberDef *m, PyObject *v); /* PyMember_SetOne: 0 / -1 */ int tp_setattr(PyObject *obj, PyObject *name, PyObject *v); /* STORE_ATTR (PyObject_SetAttr): 0 / -1 */ PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); - /* cls->tp_alloc(cls, 0) then set each slot (new references); NULL + MemoryError */ + /* cls->tp_alloc(cls, 0) then a release store of a new reference into each slot + (the fresh object is unreachable until it returns, #125); NULL + MemoryError */ int tp_is_exact(PyObject *obj, PyTypeObject *cls); /* Py_IS_TYPE */ FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 8b9f54d8e..845f42ec9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2756,7 +2756,22 @@ static inline int tp_setattr(PyObject *obj, PyObject *name, PyObject *v) /* The observable result of a trivial __init__: allocate with cls's tp_alloc and set slots[i] to * values[i] (new references; the caller keeps its own). NULL with the error set on failure; the - * half-built object is released (its unset slots are NULL, which dealloc handles). */ + * half-built object is released (its unset slots are NULL, which dealloc handles). + * + * Issue #125: each slot is filled by a release store of a new reference, which is what + * PyMember_SetOne does for a Py_T_OBJECT_EX slot (FT_ATOMIC_STORE_PTR_RELEASE of Py_XNewRef(v) + * under the object's critical section, then Py_XDECREF of the old value), minus the critical section + * and the old value. Why that is the same result: + * - every slot was checked at capture (tp_class_capture): Py_T_OBJECT_EX, writable, offset inside + * tp_basicsize; so SetOne's READONLY and type dispatch are decided already; + * - tp_alloc zeroed the object, so the old value is NULL and there is nothing to release; + * - nothing between tp_alloc and the last store can reach a safepoint (no call that detaches the + * thread state, no eval-breaker check), so no stop-the-world (gc.get_objects, gc.get_referrers) + * can hand the object to another thread before it is filled. The critical section, which could + * detach when contended, protected nothing here. A reader that did find it (none can) would see + * NULL or the stored value, as with SetOne's release store. + * A NULL value (the generator never passes one) keeps SetOne's result: AttributeError for deleting + * an unset Py_T_OBJECT_EX slot. */ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n) { @@ -2767,10 +2782,13 @@ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slot return NULL; } for (i = 0; i < n; i++) { - if (PyMember_SetOne((char *)obj, slots[i], values[i]) < 0) { + if (values[i] == NULL) { + PyErr_SetString(PyExc_AttributeError, slots[i]->name); Py_DECREF(obj); return NULL; } + __atomic_store_n((PyObject **)((char *)obj + slots[i]->offset), Py_NewRef(values[i]), + __ATOMIC_RELEASE); } return obj; } diff --git a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py index dfbeb0058..4858212dc 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py @@ -428,6 +428,20 @@ def test_gen_uses_only_the_runtime_helpers_for_slots(): assert "PyMember_" not in text and "tp_alloc" not in text and "->tp_dictoffset" not in text +def test_new_fixed_fills_its_fresh_object_with_direct_release_stores(): + """#125: a New allocates its object and fills every slot before control can reach a safepoint, so + no other thread holds the object yet. tp_new_fixed stores each new reference straight into the + slot (release store, as PyMember_SetOne's free-threaded store) instead of taking the member + protocol's per-object critical section for every slot.""" + from typedpython import cbuild + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + start = h.index("static inline PyObject *tp_new_fixed(") + body = h[start:h.index("\n}\n", start)] + assert "PyMember_SetOne" not in body and "CRITICAL_SECTION" not in body, body + assert "__atomic_store_n(" in body and "__ATOMIC_RELEASE" in body, body + assert "Py_NewRef(" in body, body # the slot owns its own reference + + # --- refcounts --------------------------------------------------------------------------------------------- def test_reference_counts_are_unchanged_over_many_calls(m): From 6f7b669ea28e902ccd0156fa9db3c46650de3663 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:44:19 +0900 Subject: [PATCH 20/61] TypedPython: MSVC fastmath enables FMA contraction only (#368) Windows CI failed test_fastmath_keeps_nan_checks: float_control(precise, off) is /fp:fast, which folds x != x. cl.exe has no reassociation-only switch, so on MSVC a fastmath function now contracts but does not reassociate. SPEC N-8 says so; a macro test pins it. --- docs/SPEC.md | 13 ++++++++----- .../src/main/python/typedpython/cgen.py | 8 +++++--- .../src/test/python/test_pipeline.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 14c5ece6f..e8ebb2032 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -748,12 +748,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. caller without fastmath): **this is the one path in TypedPython that is explicitly NOT CPython-identical** (a sum or fused multiply-add in a `@compiled(fastmath=True)` function may round differently from the interpreter so float reductions and FMA loops can vectorise). - The three compilers allow the same scope (#368): reassociation and FMA contraction only. Clang uses - `#pragma clang fp reassociate(on) contract(fast)`, MSVC `float_control(precise, off)` with - `fp_contract(on)`, GCC `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` - (the two extra options are what associative-math needs to take effect). None of them assumes finite + No compiler goes beyond reassociation and FMA contraction (#368). Clang uses + `#pragma clang fp reassociate(on) contract(fast)`, GCC + `optimize("associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast")` (the two extra + options are what associative-math needs to take effect), MSVC `fp_contract(on)` only: cl.exe has no + reassociation-only switch, and `float_control(precise, off)` (`/fp:fast`) also folds `x != x` + away, so on MSVC a fastmath function contracts but does not reassociate. None of them assumes finite values, so a NaN check (`x != x`, `isnan`) in a fastmath function still works; GCC's `fast-math` is - not used (`test_pipeline.py`: the macro test, and the NaN-check test run under a real GCC on Linux CI). + not used (`test_pipeline.py`: the macro test, the NaN-check test on the host compiler of every CI OS + including MSVC on Windows, and the same under a real GCC on Linux CI). `Status: implemented` for desktop host builds (`test_pipeline.py`, `test_cgen*.py`, `test_recursion.py`, `test_eval_breaker.py`, `test_fast_entry.py`, `test_free_threaded.py`; free-threaded 3.15t, #159). Not yet: shipping compiled modules in an app through diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 295484182..fe1635e79 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -758,15 +758,17 @@ def generate(module: ir.Module, source_path: Path, display_path: str | None = No w("#define TP_FASTMATH_BEGIN _Pragma(\"clang fp reassociate(on) contract(fast)\")") w("#define TP_FASTMATH_END _Pragma(\"clang fp reassociate(off) contract(off)\")") w("#elif defined(__GNUC__)") - # The same scope as clang and MSVC (#368): reassociation and FMA contraction only. + # The same scope as clang (#368): reassociation and FMA contraction only. # associative-math takes effect only with no-signed-zeros and no-trapping-math; never # fast-math, whose finite-math-only folds `x != x` and isnan away. w("#define TP_FASTMATH_BEGIN _Pragma(\"GCC push_options\") _Pragma(\"GCC optimize(" "\\\"associative-math,no-signed-zeros,no-trapping-math,fp-contract=fast\\\")\")") w("#define TP_FASTMATH_END _Pragma(\"GCC pop_options\")") w("#elif defined(_MSC_VER)") - w("#define TP_FASTMATH_BEGIN __pragma(float_control(precise, off, push)) __pragma(fp_contract(on))") - w("#define TP_FASTMATH_END __pragma(fp_contract(off)) __pragma(float_control(pop))") + # cl.exe has no reassociation-only switch: float_control(precise, off) is /fp:fast, which also + # folds `x != x` away (Windows CI, #368). So MSVC gets FMA contraction only. + w("#define TP_FASTMATH_BEGIN __pragma(fp_contract(on))") + w("#define TP_FASTMATH_END __pragma(fp_contract(off))") w("#else") w("#define TP_FASTMATH_BEGIN") w("#define TP_FASTMATH_END") diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py index dcfb7d6da..a50a4bfe5 100644 --- a/python-multiplatform-ksp/src/test/python/test_pipeline.py +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -321,3 +321,17 @@ def test_fastmath_keeps_nan_checks_under_gcc(write, tmp_path, monkeypatch): nan = float("nan") assert mod.is_nan(nan) is True assert mod.nan_count([1.0, nan, 2.0, nan]) == 2 + + +def test_msvc_fastmath_scope_is_contract_only(write): + """#368: cl.exe has no reassociation-only switch; `float_control(precise, off)` is /fp:fast and + folds `x != x` (seen on Windows CI). MSVC's fastmath macro enables FMA contraction only.""" + from typedpython import cgen, frontend, verify + + path = write("fm_msvc.py", _NAN_SRC) + proved, _ = verify.verify(frontend.lower(path)) + lines = cgen.generate(proved, path).splitlines() + i = lines.index("#elif defined(_MSC_VER)", next(k for k, l in enumerate(lines) if "TP_FASTMATH_BEGIN" in l)) + begin = next(l for l in lines[i:] if "define TP_FASTMATH_BEGIN" in l) + assert "fp_contract(on)" in begin, begin + assert "float_control" not in begin and "precise" not in begin, begin From eb17d997427b31d1df8afc6c320579bf6ec4fa4a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:49:26 +0900 Subject: [PATCH 21/61] TypedPython: emit-c CLI and manifest for pypackpack's compile slot (#25 step 1) python -m typedpython emit-c runs generate_project and writes a JSON manifest whose nativeModules / notNativeModules entries map one to one onto pypackpack's ExtensionModuleSource and NotNativeModule. The format (version 1) is fixed in docs/design/typedpython.md 4.3.2a for the Gradle task (#25 step 2). --- docs/SPEC.md | 9 ++ docs/design/typedpython.md | 46 +++++++ .../src/main/python/typedpython/__main__.py | 61 +++++++++ .../src/test/python/test_emit_c.py | 121 ++++++++++++++++++ 4 files changed, 237 insertions(+) create mode 100644 python-multiplatform-ksp/src/test/python/test_emit_c.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 13e3a29f4..43b783797 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -795,6 +795,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. only, changing toolchain flags or platform reuses every entry, the C compiler is never run, and the emitted `.c` built by `cbuild` behaves as `compile_project`'s extension. `compile_project` keeps building in process. + `python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE` (#25, step 1) runs + `generate_project` and writes a JSON manifest that maps one to one onto pypackpack's + `BundleRequest.nativeModules` (`ExtensionModuleSource`: `moduleName`, `sources`, `includeDirs`, + `defines`) and `notNativeModules` (`NotNativeModule`: `module`, `kind` `NOT_MARKED` or + `NOTHING_COMPILED`, `items`). Every developer module given is in exactly one list except a package + `__init__`, which is in neither; a compiled module's `sources` is its one `.c` and its + `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 + (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is + fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index eaf3a02d2..1c95c37ec 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -168,6 +168,52 @@ pypackpack 의 native compile slot 이 C 를 확장으로 빌드하므로 캐시 않으므로 그것들은 키에 넣지 않는다. 바뀌지 않은 모듈은 바이트가 같은 `.c` 로 재사용되어 slot 의 C → 확장 캐시도 맞는다. 테스트가 쓰는 `compile_project`(프로세스 안에서 빌드)는 그대로 둔다. +### 4.3.2a `emit-c` manifest (SPEC N-10, #25 1단계) + +Gradle 쪽(#25 2단계, python-multiplatform-gradle-plugin)은 TypedPython 을 이 CLI 로 돌리고 manifest 를 +pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행하지 않는다. + + python -m typedpython emit-c PATHS... --root DIR --cache DIR --manifest FILE + +- `PATHS`: 파일 또는 디렉터리(재귀, `typedpython check` 와 같은 규칙). `--root`: 모듈 이름의 기준 + 디렉터리(ppp 패키지의 `python/` 루트에 해당). `--cache`: `generate_project` 의 소스 → C 캐시. + `--manifest`: 쓸 파일. 상위 디렉터리는 만든다. +- 종료 코드: 0 manifest 를 씀, 1 게이트 오류(manifest 를 쓰지 않음, 원인은 stderr), 2 게이트를 실행할 수 없음. + C 컴파일러는 실행하지 않는다. + +형식(버전 1). 키 이름은 pypackpack `docs/design/compile-slot.md` 의 Kotlin 필드 이름과 같다. + +```json +{ + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/.../typedpython/runtime", + "nativeModules": [ + {"moduleName": "app.physics.nbody", "file": "app/physics/nbody.py", + "sources": ["/abs/cache/.../nbody.c"], "includeDirs": ["/abs/.../typedpython/runtime"], + "defines": {}, "status": "rebuilt", + "skippedItems": [{"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "..."}]} + ], + "notNativeModules": [ + {"module": "app.util", "file": "app/util.py", "kind": "NOT_MARKED", "items": []}, + {"module": "app.text", "file": "app/text.py", "kind": "NOTHING_COMPILED", + "items": [{"name": "f", "file": "app/text.py", "line": 2, "message": "..."}]} + ] +} +``` + +- `nativeModules[i]` → `ExtensionModuleSource(moduleName, sources, includeDirs, defines)`. `file`, `status`, + `skippedItems` 는 보고용이다(일부 함수가 인터프리트로 남는 것은 경고가 아니다, compile-slot.md §4). +- `notNativeModules[i]` → `NotNativeModule(module, kind, items)`, `items[j]` → `SkippedItem(name, file, line, message)`. + `line` 은 없으면 `null`. +- 모든 경로는 절대 경로(`/` 구분). `file` 만 `projectRoot` 기준 상대 경로다. +- 패키지 `__init__.py` 는 어느 목록에도 넣지 않는다(compile-slot.md: TypedPython 은 `__init__` 을 넘기지 않는다). + 그 밖의 모듈은 정확히 한 목록에 들어간다. 순서는 입력 파일 순서(디렉터리는 정렬)다. +- `sources` 는 캐시 안의 `.c` 를 가리킨다. 바뀌지 않은 모듈은 같은 경로, 같은 바이트다. +- 버전을 올리는 변경: 키 제거나 의미 변경. 키 추가는 버전을 올리지 않는다. + ### 4.3.3 고정 레이아웃 클래스 (SPEC N-11, M2b) **레이아웃이 CPython 에서도 고정일 때만** 고정 접근으로 컴파일한다, 사용자가 `__slots__` 를 직접 쓴 클래스. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 4db824787..f368b72c0 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -1,4 +1,5 @@ """`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` +`python -m typedpython emit-c PATHS... --root DIR [--cache DIR] --manifest FILE` Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. """ @@ -36,8 +37,17 @@ def main(argv: list[str]) -> int: help="incremental cache directory (default: build/typedpython-cache)") build.add_argument("--out", type=Path, default=None, help="copy the built extensions here (default: leave them in the cache)") + emit = commands.add_parser( + "emit-c", help="generate C per module (cached) and a manifest for pypackpack's compile slot") + emit.add_argument("paths", nargs="+", type=Path, help="files, or directories to scan recursively") + emit.add_argument("--root", type=Path, required=True, help="directory module names are relative to") + emit.add_argument("--cache", type=Path, default=Path("build/typedpython-c-cache"), + help="source -> C cache directory (default: build/typedpython-c-cache)") + emit.add_argument("--manifest", type=Path, required=True, help="the JSON manifest to write") args = parser.parse_args(argv) + if args.command == "emit-c": + return _emit_c(args.paths, args.root, args.cache, args.manifest) if args.command == "build": return _build(args.paths, args.cache, args.out) if args.command == "demo": @@ -94,6 +104,57 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: return EXIT_CLEAN +MANIFEST_FORMAT, MANIFEST_VERSION = "typedpython-emit-c", 1 + + +def _emit_c(paths: list[Path], root: Path, cache: Path, manifest: Path) -> int: + """SPEC N-10 / #25 step 1; the manifest format is docs/design/typedpython.md section 4.3.2a.""" + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to emit", file=sys.stderr) + return EXIT_FAILURE + try: + result = pipeline.generate_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + root = root.resolve() + runtime = Path(result.runtime_include).resolve().as_posix() + + def items(m: "pipeline.ModuleResult") -> list[dict]: + return [{"name": i.name, "file": i.file, "line": i.line, "message": i.message} + for i in m.skipped_items] + + native, other = [], [] + for m in result.modules: + if m.path.name == "__init__.py": + continue # never listed (pypackpack compile-slot.md) + rel = m.path.resolve().relative_to(root).as_posix() + if m.kind == "compiled": + assert m.c_source is not None + native.append({"moduleName": m.name, "file": rel, + "sources": [Path(m.c_source).resolve().as_posix()], + "includeDirs": [runtime], "defines": {}, "status": m.status, + "skippedItems": items(m)}) + else: + other.append({"module": m.name, "file": rel, + "kind": "NOT_MARKED" if m.kind == "not_marked" else "NOTHING_COMPILED", + "items": items(m)}) + doc = {"format": MANIFEST_FORMAT, "version": MANIFEST_VERSION, + "compilerVersion": pipeline.COMPILER_VERSION, "projectRoot": root.as_posix(), + "runtimeInclude": runtime, "nativeModules": native, "notNativeModules": other} + manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = manifest.with_name(manifest.name + ".tmp") + tmp.write_text(json.dumps(doc, indent=2) + "\n") + tmp.replace(manifest) + return EXIT_CLEAN + + def _demo(module: Path, module_argv: list[str], out: Path) -> int: import contextlib import importlib.util diff --git a/python-multiplatform-ksp/src/test/python/test_emit_c.py b/python-multiplatform-ksp/src/test/python/test_emit_c.py new file mode 100644 index 000000000..737137899 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_emit_c.py @@ -0,0 +1,121 @@ +"""`python -m typedpython emit-c` (#25 step 1, SPEC N-10): generate_project plus a JSON manifest that +maps onto pypackpack's BundleRequest.nativeModules / notNativeModules. Format: docs/design/typedpython.md +section 4.3.2a. + +Real Pyrefly; the C compiler is never run here. +""" +import json +import textwrap +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import cbuild +from typedpython.__main__ import main + +A = decorate_compiled(textwrap.dedent("""\ + def inc(n: int) -> int: + return n + 1 + """)) +PLAIN = "def hello() -> str:\n return 'hi'\n" +NOTHING = "@compiled\ndef f(s: str) -> str:\n return s[1:]\n" +BROKEN = "@compiled\ndef f(n: int) -> int:\n return n + 'x'\n" + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + (root / "app").mkdir(parents=True) + (root / "app" / "__init__.py").write_text("") + (root / "app" / "fast.py").write_text(A) + (root / "app" / "plain.py").write_text(PLAIN) + (root / "app" / "text.py").write_text(NOTHING) + return root + + +@pytest.fixture(autouse=True) +def no_c_compiler(monkeypatch): + def build(*a, **k): + raise AssertionError("emit-c must not run the C compiler") + monkeypatch.setattr(cbuild, "build", build) + + +def emit(project, tmp_path, *paths): + manifest = tmp_path / "out" / "manifest.json" + rc = main(["emit-c", *(str(p) for p in (paths or [project])), "--root", str(project), + "--cache", str(tmp_path / "cache"), "--manifest", str(manifest)]) + return rc, manifest + + +def test_manifest_lists_every_module_once_in_the_slot_shape(project, tmp_path): + rc, manifest = emit(project, tmp_path) + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["format"] == "typedpython-emit-c" and m["version"] == 1 + assert m["projectRoot"] == project.resolve().as_posix() + runtime = Path(m["runtimeInclude"]) + assert runtime.is_absolute() and (runtime / "tp_runtime.h").is_file() + assert m["compilerVersion"] + + native = {n["moduleName"]: n for n in m["nativeModules"]} + assert list(native) == ["app.fast"] + fast = native["app.fast"] + assert set(fast) >= {"moduleName", "sources", "includeDirs", "defines", "file", "status", "skippedItems"} + assert fast["file"] == "app/fast.py" and fast["defines"] == {} + assert fast["includeDirs"] == [m["runtimeInclude"]] + assert len(fast["sources"]) == 1 + c_file = Path(fast["sources"][0]) + assert c_file.is_absolute() and c_file.name == "fast.c" and c_file.is_file() + assert "PyInit_fast" in c_file.read_text() + + other = {n["module"]: n for n in m["notNativeModules"]} + assert list(other) == ["app.plain", "app.text"] + assert other["app.plain"] == {"module": "app.plain", "file": "app/plain.py", "kind": "NOT_MARKED", "items": []} + text = other["app.text"] + assert text["kind"] == "NOTHING_COMPILED" and text["items"] + item = text["items"][0] + assert set(item) == {"name", "file", "line", "message"} + assert item["name"] == "f" and item["file"] == "app/text.py" and isinstance(item["line"], int) + + every = [n["moduleName"] for n in m["nativeModules"]] + [n["module"] for n in m["notNativeModules"]] + assert "app" not in every and "app.__init__" not in every # a package __init__ is in neither + + +def test_a_second_run_reuses_the_same_c_files(project, tmp_path): + _, manifest = emit(project, tmp_path) + first = json.loads(manifest.read_text()) + c_file = Path(first["nativeModules"][0]["sources"][0]) + before = c_file.read_bytes() + assert first["nativeModules"][0]["status"] == "rebuilt" + rc, manifest = emit(project, tmp_path) + second = json.loads(manifest.read_text()) + assert rc == 0 + assert second["nativeModules"][0]["status"] == "reused" + assert second["nativeModules"][0]["sources"] == first["nativeModules"][0]["sources"] + assert c_file.read_bytes() == before + strip = lambda d: {**d, "nativeModules": [{**n, "status": None} for n in d["nativeModules"]]} + assert strip(second) == strip(first) + + +def test_files_given_one_by_one_keep_their_order(project, tmp_path): + rc, manifest = emit(project, tmp_path, project / "app" / "text.py", project / "app" / "plain.py") + assert rc == 0 + m = json.loads(manifest.read_text()) + assert m["nativeModules"] == [] + assert [n["module"] for n in m["notNativeModules"]] == ["app.text", "app.plain"] + + +def test_a_gate_error_exits_1_and_writes_no_manifest(project, tmp_path, capsys): + (project / "app" / "broken.py").write_text(BROKEN) + rc, manifest = emit(project, tmp_path) + assert rc == 1 + assert not manifest.exists() + assert "typedpython:" in capsys.readouterr().err + + +def test_no_python_files_exits_2(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + rc, manifest = emit(empty, tmp_path) + assert rc == 2 and not manifest.exists() From 8f27c263e384046174d36b8de582f2f80fe380ee Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:14:37 +0900 Subject: [PATCH 22/61] Feat: add TypedPythonEmitCTask and manifest v1 mapper in Gradle plugin (#25 step 2, SPEC N-10) --- docs/SPEC.md | 17 +- .../gradle/TypedPythonEmitCTask.kt | 359 ++++++++++++++++++ .../gradle/TypedPythonEmitCTaskTest.kt | 321 ++++++++++++++++ 3 files changed, 694 insertions(+), 3 deletions(-) create mode 100644 python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt create mode 100644 python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt diff --git a/docs/SPEC.md b/docs/SPEC.md index 1882bc197..76f1380db 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -792,9 +792,11 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. The Gradle task declares exact inputs and outputs so up-to-date checks and the build cache apply. Tests: changing one function body recompiles that module only; an unchanged rebuild spends ~0 s in the compile step. `Status: partial`: the per-module cache and its key are implemented in Python - (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`); - the Gradle task with declared inputs and outputs does not exist yet, because compiled modules are built - through pypackpack's compile slot (#25). + (`incremental.py`, `pipeline.compile_project`; `test_incremental.py`, `test_pipeline_incremental.py`), + and the Gradle `emit-c` task with declared inputs and outputs is implemented in + `python-multiplatform-gradle-plugin` (#25, step 2; `TypedPythonEmitCTask.kt`, + `TypedPythonEmitCTaskTest.kt`); wiring the resulting `nativeModules` / `notNativeModules` into + pypackpack's compile slot in the bundler (#25, step 3) remains. For that slot, `pipeline.generate_project` caches source to C instead of source to extension (#113): one `.c` per compiled module plus the directory of the header-only `tp_runtime.h`; its key is the source, the dependency interface hashes and the compiler version (which covers the runtime header), @@ -813,6 +815,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `includeDirs` the runtime header directory; paths are absolute. Exit status 0 (manifest written), 1 (a gate error; no manifest), 2 (the gate could not run). The C compiler is never run. The format is fixed in `docs/design/typedpython.md` §4.3.2a. Tests: `test_emit_c.py`. + `TypedPythonEmitCTask` in `python-multiplatform-gradle-plugin` (#25, step 2) runs + `python -m typedpython emit-c` with declared Gradle inputs (`@InputFiles` `sources` with + `PathSensitivity.RELATIVE` and `@SkipWhenEmpty`, `@Input` `pythonExecutable` and `@Optional @InputFiles` + `pythonPath` with `PathSensitivity.RELATIVE`) and outputs (`@OutputDirectory` `cacheDirectory` and + `@OutputFile` `manifestFile`), fails with `GradleException` naming the stderr diagnostics on exit 1 or 2, + and parses the written v1 manifest into `TypedPythonEmitCManifest` (`nativeModules` mapped to + `ExtensionModuleSource(moduleName, sources, includeDirs, defines)` and `notNativeModules` mapped to + `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Tests: + `TypedPythonEmitCTaskTest.kt`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt new file mode 100644 index 000000000..21f23326b --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -0,0 +1,359 @@ +package python.multiplatform.gradle + +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.api.tasks.TaskAction +import java.io.File + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest format). */ +const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" + +/** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ +const val TYPEDPYTHON_EMIT_C_VERSION = 1 + +/** + * One extension module's generated C and header/define metadata for pypackpack's compile slot + * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). + */ +data class ExtensionModuleSource( + val moduleName: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), +) + +/** + * One function or class that stayed interpreted, and why (`pypackpack` `docs/design/compile-slot.md` section 2). + * + * @param file path relative to `projectRoot`, `/`-separated. + * @param line 1-based line number, or `null` when unavailable. + */ +data class SkippedItem( + val name: String, + val file: String, + val line: Int?, + val message: String, +) + +/** + * One developer module for which TypedPython produced no C, and why (`pypackpack` `BundleRequest.notNativeModules`). + */ +data class NotNativeModule( + val module: String, + val kind: Kind, + val items: List = emptyList(), +) { + enum class Kind { + NOT_MARKED, + NOTHING_COMPILED, + } +} + +/** + * One entry of `nativeModules` in the `emit-c` manifest (`docs/design/typedpython.md` section 4.3.2a), + * carrying both the compile-slot fields ([toExtensionModuleSource]) and the reporting fields + * ([file], [status], [skippedItems]). + */ +data class TypedPythonNativeModuleEntry( + val moduleName: String, + val file: String, + val sources: List, + val includeDirs: List = emptyList(), + val defines: Map = emptyMap(), + val status: String, + val skippedItems: List = emptyList(), +) { + fun toExtensionModuleSource(): ExtensionModuleSource = + ExtensionModuleSource( + moduleName = moduleName, + sources = sources, + includeDirs = includeDirs, + defines = defines, + ) +} + +/** + * Parsed `typedpython-emit-c` v1 manifest (`docs/design/typedpython.md` section 4.3.2a, SPEC N-10). + */ +data class TypedPythonEmitCManifest( + val format: String, + val version: Int, + val compilerVersion: String, + val projectRoot: File, + val runtimeInclude: File, + val nativeModules: List, + val notNativeModules: List, +) { + /** Maps [nativeModules] onto pypackpack's `BundleRequest.nativeModules`. */ + fun toExtensionModuleSources(): List = + nativeModules.map { it.toExtensionModuleSource() } + + /** Maps [notNativeModules] onto pypackpack's `BundleRequest.notNativeModules`. */ + fun toNotNativeModules(): List = notNativeModules +} + +/** + * Parses a `typedpython-emit-c` v1 JSON manifest (`docs/design/typedpython.md` section 4.3.2a) + * into [TypedPythonEmitCManifest]. + * + * Unknown top-level or entry keys are ignored as specified in section 4.3.2a ("adding keys does not + * bump the version"). Unknown `format`, `version` or `notNativeModules[i].kind` fails with + * [IllegalArgumentException]. + */ +fun parseTypedPythonEmitCManifest(json: String): TypedPythonEmitCManifest { + val root = try { + JsonSlurper().parseText(json) + } catch (e: Exception) { + throw IllegalArgumentException("typedpython emit-c manifest is not valid JSON: ${e.message}", e) + } as? Map<*, *> ?: throw IllegalArgumentException("typedpython emit-c manifest is not a JSON object.") + + val format = root["format"] as? String + require(format == TYPEDPYTHON_EMIT_C_FORMAT) { + "typedpython emit-c manifest has format '$format'; expected '$TYPEDPYTHON_EMIT_C_FORMAT'." + } + val version = (root["version"] as? Number)?.toInt() + require(version == TYPEDPYTHON_EMIT_C_VERSION) { + "typedpython emit-c manifest has version '${root["version"]}'; expected version $TYPEDPYTHON_EMIT_C_VERSION." + } + + fun requireString(map: Map<*, *>, key: String, owner: String): String = + (map[key] as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.$key' is missing or empty.") + + fun parseSkippedItems(rawList: Any?, owner: String): List { + val list = rawList as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' must be a list.") + return list.mapIndexed { idx, elem -> + val item = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx]' is not an object.") + val lineVal = item["line"] + val line = when (lineVal) { + null -> null + is Number -> lineVal.toInt() + else -> throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner[$idx].line' must be an integer or null, got '$lineVal'.", + ) + } + SkippedItem( + name = requireString(item, "name", "$owner[$idx]"), + file = requireString(item, "file", "$owner[$idx]"), + line = line, + message = (item["message"] as? String) + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner[$idx].message' is missing."), + ) + } + } + + val rawNative = root["nativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'nativeModules' is missing or not a list.") + val nativeModules = rawNative.mapIndexed { idx, elem -> + val owner = "nativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val sourcesRaw = map["sources"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' is missing or not a list.") + val sources = sourcesRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.sources' contains a non-string entry.") + File(str) + } + val includeDirsRaw = map["includeDirs"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' is missing or not a list.") + val includeDirs = includeDirsRaw.map { path -> + val str = (path as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.includeDirs' contains a non-string entry.") + File(str) + } + val definesRaw = map["defines"] as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' is missing or not an object.") + val defines = definesRaw.entries.associate { (k, v) -> + val key = (k as? String)?.takeIf { it.isNotEmpty() } + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner.defines' has a blank key.") + val value = when (v) { + null -> null + is String -> v + else -> v.toString() + } + key to value + } + TypedPythonNativeModuleEntry( + moduleName = requireString(map, "moduleName", owner), + file = requireString(map, "file", owner), + sources = sources, + includeDirs = includeDirs, + defines = defines, + status = requireString(map, "status", owner), + skippedItems = parseSkippedItems(map["skippedItems"] ?: emptyList(), "$owner.skippedItems"), + ) + } + + val rawNotNative = root["notNativeModules"] as? List<*> + ?: throw IllegalArgumentException("typedpython emit-c manifest: 'notNativeModules' is missing or not a list.") + val notNativeModules = rawNotNative.mapIndexed { idx, elem -> + val owner = "notNativeModules[$idx]" + val map = elem as? Map<*, *> + ?: throw IllegalArgumentException("typedpython emit-c manifest: '$owner' is not an object.") + val kindRaw = requireString(map, "kind", owner) + val kind = try { + NotNativeModule.Kind.valueOf(kindRaw) + } catch (e: IllegalArgumentException) { + throw IllegalArgumentException( + "typedpython emit-c manifest: '$owner.kind' has unknown kind '$kindRaw'; " + + "expected one of ${NotNativeModule.Kind.values().joinToString()}.", + e, + ) + } + NotNativeModule( + module = requireString(map, "module", owner), + kind = kind, + items = parseSkippedItems(map["items"] ?: emptyList(), "$owner.items"), + ) + } + + return TypedPythonEmitCManifest( + format = format, + version = version, + compilerVersion = requireString(root, "compilerVersion", "root"), + projectRoot = File(requireString(root, "projectRoot", "root")), + runtimeInclude = File(requireString(root, "runtimeInclude", "root")), + nativeModules = nativeModules, + notNativeModules = notNativeModules, + ) +} + +/** + * Builds the `python -m typedpython emit-c` command line (`docs/design/typedpython.md` section 4.3.2a). + * Input files are sorted by their path relative to [rootDir] so the manifest module order is deterministic. + */ +fun typedPythonEmitCCommand( + pythonExecutable: String, + files: List, + rootDir: File, + cacheDir: File, + manifestFile: File, +): List { + val rootAbs = rootDir.absoluteFile + val sorted = files.map { it.absoluteFile }.sortedBy { file -> + file.relativeToOrSelf(rootAbs).invariantSeparatorsPath + } + return buildList { + add(pythonExecutable) + add("-m") + add("typedpython") + add("emit-c") + sorted.forEach { add(it.absolutePath) } + add("--root") + add(rootAbs.absolutePath) + add("--cache") + add(cacheDir.absoluteFile.absolutePath) + add("--manifest") + add(manifestFile.absoluteFile.absolutePath) + } +} + +/** + * SPEC N-10 (`python-multiplatform#25` step 2): runs `python -m typedpython emit-c` over [sources] + * and writes the per-module C cache ([cacheDirectory]) and the v1 JSON manifest ([manifestFile]) + * for pypackpack's native compile slot. + */ +abstract class TypedPythonEmitCTask : DefaultTask() { + + /** The developer `.py` files to compile to C. */ + @get:InputFiles + @get:SkipWhenEmpty + @get:IgnoreEmptyDirectories + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val sources: ConfigurableFileCollection + + /** + * The directory module names are derived relative to (`--root`, the ppp package's `python/` + * or `src/main` root). Fingerprinted through [sources]'s `PathSensitivity.RELATIVE`. + */ + @get:Internal + abstract val rootDirectory: DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + @get:Input + abstract val pythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in the interpreter. */ + @get:InputFiles + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val pythonPath: ConfigurableFileCollection + + /** `generate_project`'s incremental source-to-C cache directory (`--cache`). */ + @get:OutputDirectory + abstract val cacheDirectory: DirectoryProperty + + /** The JSON manifest written by `emit-c` (`--manifest`). */ + @get:OutputFile + abstract val manifestFile: RegularFileProperty + + init { + pythonExecutable.convention("python3") + } + + /** Parses the manifest written at [manifestFile] into [TypedPythonEmitCManifest]. */ + fun readManifest(): TypedPythonEmitCManifest = + parseTypedPythonEmitCManifest(manifestFile.get().asFile.readText()) + + @TaskAction + fun emitC() { + val pyFiles = sources.files.filter { it.isFile && it.extension == "py" } + val root = rootDirectory.get().asFile + val cache = cacheDirectory.get().asFile + val manifest = manifestFile.get().asFile + val executable = pythonExecutable.getOrElse("python3") + + val command = typedPythonEmitCCommand( + pythonExecutable = executable, + files = pyFiles, + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + logger.info("typedPythonEmitC: ${command.joinToString(" ")}") + + val work = temporaryDir.apply { mkdirs() } + val tmp = File(work, "tmp").apply { mkdirs() } + val stdoutFile = File(work, "emit-c.stdout") + val stderrFile = File(work, "emit-c.stderr") + + val pb = ProcessBuilder(command) + .directory(project.projectDir) + .redirectOutput(stdoutFile) + .redirectError(stderrFile) + pb.environment()["TMPDIR"] = tmp.absolutePath + val extraPythonPath = pythonPath.files.map { it.absolutePath }.filter { it.isNotEmpty() } + if (extraPythonPath.isNotEmpty()) { + val existing = pb.environment()["PYTHONPATH"].orEmpty() + val joined = extraPythonPath.joinToString(File.pathSeparator) + pb.environment()["PYTHONPATH"] = if (existing.isEmpty()) joined else "$joined${File.pathSeparator}$existing" + } + + val exitCode = pb.start().waitFor() + if (exitCode != 0) { + val details = listOf(stderrFile.readText().trim(), stdoutFile.readText().trim()) + .filter { it.isNotEmpty() } + .joinToString("\n") + .ifEmpty { "(no output)" } + throw GradleException("typedpython emit-c failed (exit $exitCode):\n$details") + } + } +} diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt new file mode 100644 index 000000000..23893acd3 --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -0,0 +1,321 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.api.tasks.IgnoreEmptyDirectories +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.SkipWhenEmpty +import org.gradle.testfixtures.ProjectBuilder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * SPEC N-10 (python-multiplatform#25 step 2): Gradle task running `python -m typedpython emit-c` + * with declared inputs/outputs and mapping the v1 manifest (`docs/design/typedpython.md` section 4.3.2a) + * onto pypackpack's `BundleRequest.nativeModules` (`ExtensionModuleSource`) and + * `BundleRequest.notNativeModules` (`NotNativeModule` / `SkippedItem`). + */ +class TypedPythonEmitCTaskTest { + + private val sampleManifestJson = + """ + { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "abc123def456", + "projectRoot": "/abs/proj", + "runtimeInclude": "/abs/site-packages/typedpython/runtime", + "nativeModules": [ + { + "moduleName": "app.physics.nbody", + "file": "app/physics/nbody.py", + "sources": ["/abs/cache/nbody.c"], + "includeDirs": ["/abs/site-packages/typedpython/runtime"], + "defines": {"TP_FLAG": "1", "TP_BARE": null}, + "status": "rebuilt", + "skippedItems": [ + {"name": "main", "file": "app/physics/nbody.py", "line": 30, "message": "not pure"} + ] + } + ], + "notNativeModules": [ + { + "module": "app.util", + "file": "app/util.py", + "kind": "NOT_MARKED", + "items": [] + }, + { + "module": "app.text", + "file": "app/text.py", + "kind": "NOTHING_COMPILED", + "items": [ + {"name": "f", "file": "app/text.py", "line": 2, "message": "str slice is not supported"}, + {"name": "g", "file": "app/text.py", "line": null, "message": "no line"} + ] + } + ] + } + """.trimIndent() + + @Test + fun parsesManifestV1AndMapsToBundleRequestShapes() { + val manifest = parseTypedPythonEmitCManifest(sampleManifestJson) + assertEquals(TYPEDPYTHON_EMIT_C_FORMAT, manifest.format) + assertEquals(TYPEDPYTHON_EMIT_C_VERSION, manifest.version) + assertEquals("abc123def456", manifest.compilerVersion) + assertEquals(File("/abs/proj"), manifest.projectRoot) + assertEquals(File("/abs/site-packages/typedpython/runtime"), manifest.runtimeInclude) + + assertEquals(1, manifest.nativeModules.size) + val nativeEntry = manifest.nativeModules.single() + assertEquals("app.physics.nbody", nativeEntry.moduleName) + assertEquals("app/physics/nbody.py", nativeEntry.file) + assertEquals(listOf(File("/abs/cache/nbody.c")), nativeEntry.sources) + assertEquals(listOf(File("/abs/site-packages/typedpython/runtime")), nativeEntry.includeDirs) + assertEquals(mapOf("TP_FLAG" to "1", "TP_BARE" to null), nativeEntry.defines) + assertEquals("rebuilt", nativeEntry.status) + assertEquals( + listOf(SkippedItem("main", "app/physics/nbody.py", 30, "not pure")), + nativeEntry.skippedItems, + ) + + val mappedNative = manifest.toExtensionModuleSources() + assertEquals( + listOf( + ExtensionModuleSource( + moduleName = "app.physics.nbody", + sources = listOf(File("/abs/cache/nbody.c")), + includeDirs = listOf(File("/abs/site-packages/typedpython/runtime")), + defines = mapOf("TP_FLAG" to "1", "TP_BARE" to null), + ), + ), + mappedNative, + ) + + val mappedNotNative = manifest.toNotNativeModules() + assertEquals(2, mappedNotNative.size) + assertEquals( + NotNativeModule( + module = "app.util", + kind = NotNativeModule.Kind.NOT_MARKED, + items = emptyList(), + ), + mappedNotNative[0], + ) + assertEquals( + NotNativeModule( + module = "app.text", + kind = NotNativeModule.Kind.NOTHING_COMPILED, + items = listOf( + SkippedItem("f", "app/text.py", 2, "str slice is not supported"), + SkippedItem("g", "app/text.py", null, "no line"), + ), + ), + mappedNotNative[1], + ) + assertNull(mappedNotNative[1].items[1].line) + } + + @Test + fun AdditiveTopLevelOrEntryKeysDoNotBreakVersion1Parsing() { + val withExtraKeys = sampleManifestJson.replace( + "\"version\": 1,", + "\"version\": 1, \"futureTopKey\": 42,", + ) + val manifest = parseTypedPythonEmitCManifest(withExtraKeys) + assertEquals(1, manifest.nativeModules.size) + assertEquals(2, manifest.notNativeModules.size) + } + + @Test + fun rejectsUnknownFormatVersionOrKind() { + val wrongFormat = sampleManifestJson.replace("\"typedpython-emit-c\"", "\"other-format\"") + val formatErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongFormat) } + assertTrue("typedpython-emit-c" in formatErr.message.orEmpty()) + + val wrongVersion = sampleManifestJson.replace("\"version\": 1", "\"version\": 2") + val versionErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongVersion) } + assertTrue("version" in versionErr.message.orEmpty()) + + val wrongKind = sampleManifestJson.replace("\"NOT_MARKED\"", "\"UNKNOWN_KIND\"") + val kindErr = assertFailsWith { parseTypedPythonEmitCManifest(wrongKind) } + assertTrue("UNKNOWN_KIND" in kindErr.message.orEmpty()) + } + + @Test + fun buildsDeterministicCommandLineWithSortedInputFiles() { + val root = File("/proj/src/main") + val cache = File("/proj/build/typedpython/c-cache") + val manifest = File("/proj/build/typedpython/emit-c-manifest.json") + val b = File("/proj/src/main/app/b.py") + val a = File("/proj/src/main/app/a.py") + + val cmd = typedPythonEmitCCommand( + pythonExecutable = "/venv/bin/python3", + files = listOf(b, a), + rootDir = root, + cacheDir = cache, + manifestFile = manifest, + ) + assertEquals( + listOf( + "/venv/bin/python3", + "-m", + "typedpython", + "emit-c", + a.absolutePath, + b.absolutePath, + "--root", + root.absolutePath, + "--cache", + cache.absolutePath, + "--manifest", + manifest.absolutePath, + ), + cmd, + ) + } + + @Test + fun taskDeclaresGradleInputsAndOutputsForIncrementalAndBuildCacheSupport() { + val getters = TypedPythonEmitCTask::class.java.methods.associateBy { it.name } + + val getSources = getters.getValue("getSources") + assertTrue(getSources.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getSources.isAnnotationPresent(SkipWhenEmpty::class.java)) + assertTrue(getSources.isAnnotationPresent(IgnoreEmptyDirectories::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getSources.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getRootDirectory").isAnnotationPresent(Internal::class.java)) + assertTrue(getters.getValue("getPythonExecutable").isAnnotationPresent(Input::class.java)) + + val getPythonPath = getters.getValue("getPythonPath") + assertTrue(getPythonPath.isAnnotationPresent(InputFiles::class.java)) + assertTrue(getPythonPath.isAnnotationPresent(Optional::class.java)) + assertEquals( + PathSensitivity.RELATIVE, + getPythonPath.getAnnotation(PathSensitive::class.java)?.value, + ) + + assertTrue(getters.getValue("getCacheDirectory").isAnnotationPresent(OutputDirectory::class.java)) + assertTrue(getters.getValue("getManifestFile").isAnnotationPresent(OutputFile::class.java)) + } + + @Test + fun taskActionExecutesStubEmitCAndParsesResultingManifest() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef inc(n: int) -> int:\n return n + 1\n") + } + val fakeStubRoot = project.file("fake-pythonpath").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + val runtimeDir = File(tpPkg, "runtime").apply { mkdirs() } + File(runtimeDir, "tp_runtime.h").writeText("/* header */\n") + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json, sys + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitTypedPythonC", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + task.emitC() + + val parsed = task.readManifest() + assertEquals(1, parsed.nativeModules.size) + val mapped = parsed.toExtensionModuleSources().single() + assertEquals("app.fast", mapped.moduleName) + assertTrue(mapped.sources.single().isFile) + assertEquals("fast.c", mapped.sources.single().name) + } + + @Test + fun taskActionFailsWithDiagnosticsOnExit1Or2() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/broken.py").apply { + parentFile.mkdirs() + writeText("def f(x): return x\n") + } + val fakeStubRoot = project.file("fake-failing-tp").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import sys + print("typedpython: broken.py does not pass the gate in compiled mode", file=sys.stderr) + sys.exit(1) + """.trimIndent() + "\n", + ) + + val task = project.tasks.register("emitBroken", TypedPythonEmitCTask::class.java) { + sources.from(pyFile) + rootDirectory.set(pkgRoot) + pythonExecutable.set("python3") + pythonPath.from(fakeStubRoot) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) + }.get() + + val err = assertFailsWith { task.emitC() } + assertTrue("exit 1" in err.message.orEmpty(), err.message) + assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) + } +} From 3c2ed3d3217ab8328501e389f444e69518126578 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:27:26 +0900 Subject: [PATCH 23/61] Feat(gradle-plugin): register emitTypedPythonC from pythonBindings DSL (SPEC N-10, #25) --- docs/SPEC.md | 8 +- .../gradle/PythonBindingsPlugin.kt | 47 ++++++++++ .../gradle/TypedPythonEmitCTask.kt | 3 + .../gradle/TypedPythonEmitCTaskTest.kt | 86 +++++++++++++++++++ .../fixtures/app/build.gradle.kts | 19 ++++ .../fixture/app/TypedPythonKotlinCallTest.kt | 13 +++ 6 files changed, 174 insertions(+), 2 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 76f1380db..90d07e494 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -822,8 +822,12 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `@OutputFile` `manifestFile`), fails with `GradleException` naming the stderr diagnostics on exit 1 or 2, and parses the written v1 manifest into `TypedPythonEmitCManifest` (`nativeModules` mapped to `ExtensionModuleSource(moduleName, sources, includeDirs, defines)` and `notNativeModules` mapped to - `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Tests: - `TypedPythonEmitCTaskTest.kt`. + `NotNativeModule(module, kind, items)` with `SkippedItem(name, file, line, message)`). Setting + `pythonBindings.typedPythonSourceDir` registers `emitTypedPythonC` (`TypedPythonEmitCTask`, with + `typedPythonRootDir` defaulting to `typedPythonSourceDir`, `typedPythonExecutable` defaulting to + `"python3"`, optional `typedPythonPath`, `cacheDirectory` at `build/typedpython/c-cache` and + `manifestFile` at `build/typedpython/emit-c-manifest.json`); leaving `typedPythonSourceDir` unset + registers no task. Tests: `TypedPythonEmitCTaskTest.kt`, `TypedPythonKotlinCallTest.kt`. - **N-11** TypedPython fixed-layout classes (M2b, 2026-11-21; design §4.3.3). A class compiles to fixed field access only when its layout is fixed **in CPython too**: it declares `__slots__` listing exactly its annotated instance fields, has no base but `object`, no metaclass, no decorator but `@compiled`, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index 823db419d..895f42e4e 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -241,6 +241,25 @@ interface PythonBindingsExtension { /** The `ios { }` block itself: without it the nested property is reachable only as `ios.x`. */ fun ios(action: org.gradle.api.Action) = action.execute(ios) + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): directory of developer `.py` files to compile + * to C with `python -m typedpython emit-c`. When set, registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]); unset (the default) registers no task. + */ + val typedPythonSourceDir: org.gradle.api.file.DirectoryProperty + + /** + * The directory module names are derived relative to (`--root`). Defaults to + * [typedPythonSourceDir]. + */ + val typedPythonRootDir: org.gradle.api.file.DirectoryProperty + + /** The Python interpreter executable that runs `-m typedpython emit-c` (defaults to `"python3"`). */ + val typedPythonExecutable: Property + + /** Optional `PYTHONPATH` entries for locating the `typedpython` package when not installed in [typedPythonExecutable]. */ + val typedPythonPath: org.gradle.api.file.ConfigurableFileCollection } /** @@ -326,6 +345,34 @@ class PythonBindingsPlugin : Plugin { // `plugins { ... }` and so do not exist while this plugin is being applied. configureArtifactBindings(this, extension) configureWasmProxyExports(this) + configureTypedPythonEmitC(this, extension) + } + } + + /** + * SPEC N-10 (`python-multiplatform#25` step 2): registers [EMIT_TYPEDPYTHON_C_TASK] + * ([TypedPythonEmitCTask]) when [PythonBindingsExtension.typedPythonSourceDir] is set. + * + * A no-op when [PythonBindingsExtension.typedPythonSourceDir] is unset, so projects that do not + * compile developer Python modules with TypedPython get no task. + */ + internal fun configureTypedPythonEmitC(project: Project, extension: PythonBindingsExtension) { + if (!extension.typedPythonSourceDir.isPresent) return + val srcDir = extension.typedPythonSourceDir.get().asFile + project.tasks.register(EMIT_TYPEDPYTHON_C_TASK, TypedPythonEmitCTask::class.java) { + group = "python" + description = "Runs python -m typedpython emit-c over pythonBindings.typedPythonSourceDir and writes the compile-slot manifest." + sources.from( + project.fileTree(srcDir) { + include("**/*.py") + exclude("**/__pycache__/**", "**/.*", "**/build/**") + }, + ) + rootDirectory.set(extension.typedPythonRootDir.orElse(extension.typedPythonSourceDir)) + pythonExecutable.set(extension.typedPythonExecutable.orElse("python3")) + pythonPath.from(extension.typedPythonPath) + cacheDirectory.set(project.layout.buildDirectory.dir("typedpython/c-cache")) + manifestFile.set(project.layout.buildDirectory.file("typedpython/emit-c-manifest.json")) } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt index 21f23326b..7e5d2e28f 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/TypedPythonEmitCTask.kt @@ -26,6 +26,9 @@ const val TYPEDPYTHON_EMIT_C_FORMAT = "typedpython-emit-c" /** `docs/design/typedpython.md` section 4.3.2a (`emit-c` manifest version). */ const val TYPEDPYTHON_EMIT_C_VERSION = 1 +/** Task name registered by [PythonBindingsPlugin] when [PythonBindingsExtension.typedPythonSourceDir] is set. */ +const val EMIT_TYPEDPYTHON_C_TASK = "emitTypedPythonC" + /** * One extension module's generated C and header/define metadata for pypackpack's compile slot * (`pypackpack` `ExtensionCompiler.kt`, `BundleRequest.nativeModules`). diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt index 23893acd3..2ff23ddba 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/TypedPythonEmitCTaskTest.kt @@ -318,4 +318,90 @@ class TypedPythonEmitCTaskTest { assertTrue("exit 1" in err.message.orEmpty(), err.message) assertTrue("broken.py does not pass the gate" in err.message.orEmpty(), err.message) } + + @Test + fun pluginDoesNotRegisterEmitTypedPythonCWhenSourceDirIsUnset() { + val project = ProjectBuilder.builder().build() + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + assertEquals(null, project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK)) + } + + @Test + fun pluginRegistersEmitTypedPythonCFromExtensionAndExecutesIt() { + val project = ProjectBuilder.builder().build() + val pkgRoot = project.file("pkg/src/main").apply { mkdirs() } + val pyFile = File(pkgRoot, "app/fast.py").apply { + parentFile.mkdirs() + writeText("@compiled\ndef f(x: int) -> int:\n return x + 1\n") + } + File(pkgRoot, "app/__init__.pyi").writeText("") + val fakeStubRoot = project.file("fake-tp-plugin").apply { mkdirs() } + val tpPkg = File(fakeStubRoot, "typedpython").apply { mkdirs() } + File(tpPkg, "__init__.py").writeText("") + File(tpPkg, "__main__.py").writeText( + """ + import argparse, json + from pathlib import Path + p = argparse.ArgumentParser() + p.add_argument("cmd") + p.add_argument("paths", nargs="+") + p.add_argument("--root", required=True) + p.add_argument("--cache", required=True) + p.add_argument("--manifest", required=True) + a = p.parse_args() + assert all(not x.endswith(".pyi") for x in a.paths), a.paths + cache = Path(a.cache).resolve() + cache.mkdir(parents=True, exist_ok=True) + c_file = cache / "fast.c" + c_file.write_text("/* generated C */\n") + runtime = (Path(__file__).resolve().parent / "runtime").as_posix() + doc = { + "format": "typedpython-emit-c", + "version": 1, + "compilerVersion": "stub-1", + "projectRoot": Path(a.root).resolve().as_posix(), + "runtimeInclude": runtime, + "nativeModules": [{ + "moduleName": "app.fast", + "file": "app/fast.py", + "sources": [c_file.as_posix()], + "includeDirs": [runtime], + "defines": {}, + "status": "rebuilt", + "skippedItems": [], + }], + "notNativeModules": [], + } + mf = Path(a.manifest) + mf.parent.mkdir(parents=True, exist_ok=True) + mf.write_text(json.dumps(doc)) + """.trimIndent() + "\n", + ) + + val extension = project.extensions.create("pythonBindings", PythonBindingsExtension::class.java) + extension.typedPythonSourceDir.set(pkgRoot) + extension.typedPythonExecutable.set("python3") + extension.typedPythonPath.from(fakeStubRoot) + + PythonBindingsPlugin().configureTypedPythonEmitC(project, extension) + + val task = project.tasks.findByName(EMIT_TYPEDPYTHON_C_TASK) as? TypedPythonEmitCTask + ?: error("expected $EMIT_TYPEDPYTHON_C_TASK to be registered by PythonBindingsPlugin") + assertEquals(listOf(pyFile.canonicalFile), task.sources.files.map { it.canonicalFile }) + assertEquals(pkgRoot.canonicalFile, task.rootDirectory.get().asFile.canonicalFile) + assertEquals( + project.layout.buildDirectory.dir("typedpython/c-cache").get().asFile.canonicalFile, + task.cacheDirectory.get().asFile.canonicalFile, + ) + assertEquals( + project.layout.buildDirectory.file("typedpython/emit-c-manifest.json").get().asFile.canonicalFile, + task.manifestFile.get().asFile.canonicalFile, + ) + + task.emitC() + val parsed = task.readManifest() + assertEquals("app.fast", parsed.toExtensionModuleSources().single().moduleName) + } } diff --git a/python-multiplatform-ksp/fixtures/app/build.gradle.kts b/python-multiplatform-ksp/fixtures/app/build.gradle.kts index 57ec527d8..9cd4ac650 100644 --- a/python-multiplatform-ksp/fixtures/app/build.gradle.kts +++ b/python-multiplatform-ksp/fixtures/app/build.gradle.kts @@ -106,6 +106,19 @@ val typedPythonSkipReason: String? = when { else -> null } +pythonBindings { + typedPythonSourceDir.set(file("src/desktopTest/typedpython")) + typedPythonExecutable.set(typedPythonHost?.absolutePath ?: "python3") + typedPythonPath.from(rootDir.resolve("python-multiplatform-ksp/src/main/python")) +} + +val typedPythonEmitCManifest = layout.buildDirectory.file("typedpython/emit-c-manifest.json") + +tasks.matching { it.name == "emitTypedPythonC" }.configureEach { + val skip = typedPythonSkipReason + onlyIf("TypedPython emit-c: ${skip ?: "enabled"}") { skip == null } +} + @Suppress("UNCHECKED_CAST") val typedPythonIncludeDir: Provider? = typedPythonMacTarget?.let { target -> val includes = project(":python-multiplatform").extensions.getByName("cpythonIncludeDirectories") @@ -156,6 +169,7 @@ val buildTypedPythonBridge by tasks.registering(Exec::class) { tasks.named("desktopTest") { dependsOn(buildTypedPythonBridge) + dependsOn("emitTypedPythonC") // The tests import tp_kotlin_bridge.py from src/; without this the interpreter leaves a // __pycache__ in the source tree. environment("PYTHONDONTWRITEBYTECODE", "1") @@ -165,10 +179,15 @@ tasks.named("desktopTest") { } if (typedPythonSkipReason == null) { inputs.dir(typedPythonBridgeDir).withPropertyName("typedPythonBridge") + inputs.file(typedPythonEmitCManifest).withPropertyName("typedPythonEmitCManifest") systemProperty( "typedpython.bridge.properties", typedPythonBridgeDir.get().file("bridge.properties").asFile.absolutePath, ) + systemProperty( + "typedpython.emitC.manifest", + typedPythonEmitCManifest.get().asFile.absolutePath, + ) } else { systemProperty("typedpython.bridge.skip", typedPythonSkipReason) } diff --git a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt index 101d0d7a6..4a45e5e62 100644 --- a/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt +++ b/python-multiplatform-ksp/fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt @@ -291,6 +291,19 @@ class TypedPythonKotlinCallTest { ) } + @Test + fun emitTypedPythonCTaskWritesManifestForTpKotlinBridge() { + val manifestPath = System.getProperty("typedpython.emitC.manifest") + assertTrue(manifestPath != null, "system property typedpython.emitC.manifest is unset") + val manifestFile = File(manifestPath!!) + assertTrue(manifestFile.isFile, "emitTypedPythonC ran but $manifestFile does not exist") + val text = manifestFile.readText() + assertTrue("\"format\": \"typedpython-emit-c\"" in text, text) + assertTrue("\"moduleName\": \"tp_kotlin_bridge\"" in text, text) + assertTrue("tp_kotlin_bridge.c" in text, text) + assertTrue("\"notNativeModules\": []" in text, text) + } + /** * The watched reference counts once the JVM has collected the `PyObject` wrappers the binder made * and their cleaners have handed the references back. From e01d4223e6e0d5e44e7140f855464ea0b060835d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:42:26 +0900 Subject: [PATCH 24/61] Publish the library, KSP processor and Gradle plugin to Maven Central (#375) Shared gradle/central-publishing.gradle.kts adds POM fields, javadoc and sources jars, in-memory signing when a key is present, and a centralStaging repository. maven-central.yml bundles on macOS and uploads to the Central Portal on a version tag; a pull request touching the setup runs a dry run signed with a throwaway key. desktopNativeMain isDirectory uses opendir instead of stat: st_mode differs in width between macOS and Linux and the shared metadata compilation that publishing runs rejected it. --- .github/scripts/release/central-bundle.sh | 75 +++++++++ .github/scripts/release/central-upload.sh | 68 ++++++++ .github/workflows/maven-central.yml | 140 +++++++++++++++++ build.gradle.kts | 13 ++ docs/guide/getting-started.html | 19 ++- docs/roadmap/ROADMAP.md | 15 +- gradle/central-publishing.gradle.kts | 148 ++++++++++++++++++ .../build.gradle.kts | 2 + python-multiplatform-ksp/build.gradle.kts | 2 + python-multiplatform/build.gradle.kts | 42 +---- .../env/PackagedPythonHome.desktopNative.kt | 22 +-- 11 files changed, 491 insertions(+), 55 deletions(-) create mode 100755 .github/scripts/release/central-bundle.sh create mode 100755 .github/scripts/release/central-upload.sh create mode 100644 .github/workflows/maven-central.yml create mode 100644 gradle/central-publishing.gradle.kts diff --git a/.github/scripts/release/central-bundle.sh b/.github/scripts/release/central-bundle.sh new file mode 100755 index 000000000..55e8f8f2a --- /dev/null +++ b/.github/scripts/release/central-bundle.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Builds the Maven Central deployment of python-multiplatform: every component, signed, in one zip. +# .github/scripts/release/central-bundle.sh [--throwaway-key] +# Run from the repository root. Reads MAVEN_GPG_PRIVATE_KEY and MAVEN_GPG_PASSPHRASE (Gradle signs +# in memory, gradle/central-publishing.gradle.kts). --throwaway-key makes a key for this run only, +# for a dry run that must prove signing works without the release key. Without either, the bundle is +# unsigned and the script says so; Central would refuse it. +# Writes build/central-staging (a Maven layout) and ; uploads nothing, see central-upload.sh. +set -euo pipefail +out="${1:?usage: central-bundle.sh [--throwaway-key]}" +mode="${2:-}" +root="$(pwd)" +staging="$root/build/central-staging" +version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + +# Not under the repository: gpg-agent's socket path must stay under about 100 characters. +verify_home="$(mktemp -d)" +chmod 700 "$verify_home" +trap 'rm -rf "$verify_home"' EXIT + +if [[ "$mode" == "--throwaway-key" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --passphrase '' --quick-generate-key \ + "python-multiplatform dry run " rsa3072 sign 1d + MAVEN_GPG_PRIVATE_KEY="$(GNUPGHOME="$verify_home" gpg --batch --pinentry-mode loopback --passphrase '' --armor --export-secret-keys)" + export MAVEN_GPG_PRIVATE_KEY MAVEN_GPG_PASSPHRASE="" +elif [[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]]; then + GNUPGHOME="$verify_home" gpg --batch --quiet --import <<< "$MAVEN_GPG_PRIVATE_KEY" +fi +signed=false +[[ -n "${MAVEN_GPG_PRIVATE_KEY:-}" ]] && signed=true + +mkdir -p "$root/.cache" +rm -rf "$staging" +# Gradle reads the key from the environment; the plugin build is a separate build but inherits it. +./gradlew --console=plain publishAllToCentralStaging 2>&1 | tee "$root/.cache/central-gradle.log" || { echo "error: gradle failed (pipe status ${PIPESTATUS[0]})" >&2; exit 1; } + +# What Central checks first: every module has pom + jar/klib/zip, sources and javadoc where it is a +# jar publication, and (signed) a good .asc beside every file. +fail=0 +while IFS= read -r pom; do + dir="$(dirname "$pom")"; base="$(basename "$pom" .pom)" + ver="$(basename "$dir")"; art="$(basename "$(dirname "$dir")")" + [[ "$ver" == "$version" ]] || { echo "error: $pom is not at $version" >&2; fail=1; } + grep -q 'https://github.com/thisisthepy/python-multiplatform' "$pom" || { echo "error: $pom has no project url" >&2; fail=1; } + grep -q '' "$pom" && grep -q '' "$pom" && grep -q '' "$pom" || { echo "error: $pom lacks licenses, developers or scm" >&2; fail=1; } + if ! grep -q 'pom' "$pom"; then + [[ -e "$dir/$base-javadoc.jar" ]] || { echo "error: $art has no javadoc jar" >&2; fail=1; } + [[ -e "$dir/$base-sources.jar" ]] || { echo "error: $art has no sources jar" >&2; fail=1; } + fi +done < <(find "$staging" -name '*.pom') +if $signed; then + while IFS= read -r f; do + case "$f" in *.asc|*.md5|*.sha1|*.sha256|*.sha512) continue ;; esac + [[ -e "$f.asc" ]] || { echo "error: $f is not signed" >&2; fail=1; continue; } + GNUPGHOME="$verify_home" gpg --batch --quiet --verify "$f.asc" "$f" 2>/dev/null || { echo "error: bad signature on $f" >&2; fail=1; } + done < <(find "$staging" -type f -not -name 'maven-metadata*') +else + echo "warning: no signing key, the bundle is unsigned and Central will refuse it" >&2 +fi +(( fail == 0 )) || exit 1 + +echo "== left out of the bundle (not built on this machine)" +grep -h "CENTRAL-SKIPPED" "$root/.cache/central-gradle.log" || echo "(none)" +echo "== coordinates in the bundle" +find "$staging" -name '*.pom' | while IFS= read -r pom; do + rel="${pom#"$staging"/}"; ver="$(basename "$(dirname "$rel")")"; art="$(basename "$(dirname "$(dirname "$rel")")")" + grp="$(dirname "$(dirname "$(dirname "$rel")")" | tr / .)" + echo "$grp:$art:$ver" +done | sort + +mkdir -p "$(dirname "$out")" +out="$(cd "$(dirname "$out")" && pwd)/$(basename "$out")" +rm -f "$out" +(cd "$staging" && zip -qr "$out" . -x 'maven-metadata*' '*/maven-metadata*') +echo "== bundle $out ($(find "$staging" -type f | wc -l | tr -d ' ') files)" diff --git a/.github/scripts/release/central-upload.sh b/.github/scripts/release/central-upload.sh new file mode 100755 index 000000000..39c3a85e1 --- /dev/null +++ b/.github/scripts/release/central-upload.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Uploads a bundle made by central_bundle.py to the Sonatype Central Portal and waits for its +# validation. +# +# Usage: central-upload.sh [USER_MANAGED|AUTOMATIC] +# +# Reads CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD, the two halves of a Central Portal +# user token (central.sonatype.com, View Account, Generate User Token). The Portal API takes +# them as a bearer token of base64(":"). +# +# USER_MANAGED (the default) stops after validation: the deployment waits in the Portal under +# Deployments until someone presses Publish or Drop, so a person looks at it before it becomes +# permanent. AUTOMATIC publishes as soon as validation passes. Either way nothing published to +# Central can be changed or removed afterwards. +# +# Exits 0 when the deployment is VALIDATED (USER_MANAGED) or PUBLISHING/PUBLISHED (AUTOMATIC), +# 1 when it FAILED, printing the Portal's errors, and 2 on a usage or network error. +set -euo pipefail + +API="https://central.sonatype.com/api/v1/publisher" + +bundle="${1:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +name="${2:?usage: central-upload.sh [USER_MANAGED|AUTOMATIC]}" +type="${3:-USER_MANAGED}" +case "$type" in USER_MANAGED|AUTOMATIC) ;; *) echo "error: publishing type '$type'" >&2; exit 2 ;; esac +[[ -s "$bundle" ]] || { echo "error: no bundle at $bundle" >&2; exit 2; } +[[ -n "${CENTRAL_TOKEN_USERNAME:-}" && -n "${CENTRAL_TOKEN_PASSWORD:-}" ]] || { + echo "error: CENTRAL_TOKEN_USERNAME and CENTRAL_TOKEN_PASSWORD must hold a Central Portal user token" >&2 + exit 2 +} + +token="$(printf '%s:%s' "$CENTRAL_TOKEN_USERNAME" "$CENTRAL_TOKEN_PASSWORD" | base64 | tr -d '\n')" +auth="Authorization: Bearer $token" + +encoded_name="$(python3 -I -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$name")" +echo "==> uploading $bundle as '$name' ($type)" +id="$(curl --fail-with-body --silent --show-error --retry 3 \ + --header "$auth" \ + --form "bundle=@$bundle" \ + "$API/upload?publishingType=$type&name=$encoded_name")" +echo "deployment $id" + +# Validation of a large bundle takes minutes; give it an hour. +for _ in $(seq 1 120); do + status="$(curl --fail-with-body --silent --show-error --retry 3 --request POST \ + --header "$auth" "$API/status?id=$id")" + state="$(python3 -I -c 'import json, sys; print(json.load(sys.stdin)["deploymentState"])' <<< "$status")" + echo "state $state" + case "$state" in + VALIDATED) + if [[ "$type" == USER_MANAGED ]]; then + echo "validated; publish or drop it at https://central.sonatype.com/publishing/deployments" + exit 0 + fi + ;; + PUBLISHING|PUBLISHED) + echo "publishing; it appears on https://repo1.maven.org/maven2/org/thisisthepy/ within about 30 minutes" + exit 0 + ;; + FAILED) + python3 -I -c 'import json, sys; print(json.dumps(json.load(sys.stdin).get("errors"), indent=2))' <<< "$status" >&2 + exit 1 + ;; + esac + sleep 30 +done +echo "error: deployment $id is still $state after an hour; see https://central.sonatype.com/publishing/deployments" >&2 +exit 2 diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml new file mode 100644 index 000000000..e79994d0b --- /dev/null +++ b/.github/workflows/maven-central.yml @@ -0,0 +1,140 @@ +name: Maven Central + +# Publishes python-multiplatform (library for every target, KSP processor, Gradle plugin and its +# plugin marker) to Maven Central through the Sonatype Central Portal, as ONE deployment. +# +# - A version tag (v-, the tag desktop-release.yml also reads) +# builds, signs with the release key and uploads. +# - A manual run is a dry run unless "dry_run" is turned off: it builds and signs with a key made in +# the job and thrown away, checks the bundle and keeps it as a workflow artifact. Nothing is +# uploaded. With dry_run off, pick the tag to publish in "tag". +# +# macOS is required: the Apple targets (iosArm64, iosSimulatorArm64, iosX64, macosArm64) only build +# there. Secrets read by a real upload, nothing else reads them: +# MAVEN_GPG_PRIVATE_KEY signing key, ASCII armoured (gpg --armor --export-secret-keys) +# MAVEN_GPG_PASSPHRASE its passphrase +# CENTRAL_TOKEN_USERNAME Central Portal user token, username half +# CENTRAL_TOKEN_PASSWORD Central Portal user token, password half +# The upload runs in the environment "maven-central", where required reviewers can be set. + +on: + push: + tags: + - 'v*' + # A pull request that touches the publishing setup runs the dry run, so the bundle is checked + # before the first tag depends on it. + pull_request: + paths: + - '.github/workflows/maven-central.yml' + - '.github/scripts/release/central-*.sh' + - 'gradle/central-publishing.gradle.kts' + workflow_dispatch: + inputs: + dry_run: + description: 'Build and sign with a throwaway key, upload nothing' + type: boolean + default: true + tag: + description: 'Tag to publish when dry_run is off (e.g. v3.15.0-alpha02); ignored for a dry run' + required: false + default: '' + publishing_type: + description: 'USER_MANAGED waits in the Central Portal for someone to press Publish; AUTOMATIC publishes once validated' + type: choice + options: [USER_MANAGED, AUTOMATIC] + default: USER_MANAGED + +permissions: + contents: read + +concurrency: + group: maven-central-${{ github.event.inputs.tag || github.ref }} + cancel-in-progress: false + +jobs: + bundle: + name: Build and sign the bundle (macOS) + runs-on: macos-latest + timeout-minutes: 180 + env: + DRY_RUN: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }} + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ (github.event_name == 'push' || inputs.dry_run == false) && (github.event.inputs.tag || github.ref) || github.ref }} + # The tag's own gradle.properties decides the version, not the branch it was cut from. + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + + - name: The tag names this build's version + if: env.DRY_RUN != 'true' + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + test "$TAG" = "v$version" || { echo "error: tag $TAG does not match version $version" >&2; exit 1; } + echo "VERSION=$version" >> "$GITHUB_ENV" + + - name: Build and sign the bundle (dry run, throwaway key) + if: env.DRY_RUN == 'true' + run: .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" --throwaway-key + + - name: Build and sign the bundle (release key) + if: env.DRY_RUN != 'true' + env: + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + set -euo pipefail + [[ -n "$MAVEN_GPG_PRIVATE_KEY" ]] || { echo "::error::the MAVEN_GPG_PRIVATE_KEY secret is not set"; exit 1; } + .github/scripts/release/central-bundle.sh "$RUNNER_TEMP/central-bundle.zip" + + - name: Keep the bundle + uses: actions/upload-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }}/central-bundle.zip + retention-days: 14 + + + upload: + name: Upload to the Central Portal + needs: bundle + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.dry_run == false) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + # Required reviewers can be set on this environment, so even a pushed tag waits for a person + # before anything reaches Central. A dry run never gets here. + environment: maven-central + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.tag || github.ref }} + sparse-checkout: | + .github/scripts/release + gradle.properties + sparse-checkout-cone-mode: false + + - uses: actions/download-artifact@v4 + with: + name: central-bundle + path: ${{ runner.temp }} + + - name: Upload + env: + CENTRAL_TOKEN_USERNAME: ${{ secrets.CENTRAL_TOKEN_USERNAME }} + CENTRAL_TOKEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN_PASSWORD }} + TYPE: ${{ inputs.publishing_type || 'USER_MANAGED' }} + run: | + set -euo pipefail + version="$(sed -nE 's/^pythonVersion=(.*)$/\1/p' gradle.properties)-$(sed -nE 's/^libraryVersionSuffix=(.*)$/\1/p' gradle.properties)" + .github/scripts/release/central-upload.sh "$RUNNER_TEMP/central-bundle.zip" \ + "python-multiplatform $version" "$TYPE" diff --git a/build.gradle.kts b/build.gradle.kts index 63c01e66a..cad457f0d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,19 @@ tasks.register("publishAllToMavenLocal") { dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) } +// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with all three +// components. Signed when MAVEN_GPG_PRIVATE_KEY is set (gradle/central-publishing.gradle.kts). +// It does not clean the directory (the plugin build is another build, so ordering cannot reach +// it): .github/scripts/release/central-bundle.sh empties it first. +tasks.register("publishAllToCentralStaging") { + group = "publishing" + description = "Publishes python-multiplatform, python-multiplatform-ksp and " + + "python-multiplatform-gradle-plugin (with its plugin marker) to build/central-staging." + dependsOn(":python-multiplatform:publishAllToCentralStaging") + dependsOn(":python-multiplatform-ksp:publishAllToCentralStaging") + dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishAllToCentralStaging")) +} + // The build's single Kotlin/Wasm npm lock (one workspace for every wasm module) lives next to the // version catalog instead of a root `kotlin-js-store/` (issue #110). KGP only allows this on the // root project; the `*Property` forms are the non-deprecated API from Kotlin 2.4 on. Mismatch diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html index 61f3d1b07..dd10ff835 100644 --- a/docs/guide/getting-started.html +++ b/docs/guide/getting-started.html @@ -69,7 +69,24 @@

03Your first call

There is no close() in that code, and none is missing: every wrapper releases its Python reference when the Kotlin object is collected.위 코드에는 close() 가 없고, 빠진 것도 아니다. 모든 래퍼는 Kotlin 객체가 수거될 때 Python 참조를 놓는다.

-

04Verify a change변경 검증하기

+

04Use it from Maven CentralMaven Central 에서 쓰기

+
// settings.gradle.kts
+pluginManagement { repositories { mavenCentral(); gradlePluginPortal() } }
+dependencyResolutionManagement { repositories { mavenCentral() } }
+
+// build.gradle.kts of a Kotlin Multiplatform module
+plugins {
+    kotlin("multiplatform")
+    id("org.thisisthepy.python.multiplatform.bindings") version "3.15.0-alpha02"
+}
+kotlin {
+    sourceSets.commonMain.dependencies {
+        implementation("org.thisisthepy:python-multiplatform:3.15.0-alpha02")
+    }
+}
+

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

+ +

05Verify a change변경 검증하기

./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .tmp/build.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .tmp/test.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform-ksp:fixtures:app:desktopTest --rerun
diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md
index bd328f959..029df0d52 100644
--- a/docs/roadmap/ROADMAP.md
+++ b/docs/roadmap/ROADMAP.md
@@ -300,10 +300,17 @@ stripped), the root metadata jar is 382 KB, `PythonBootstrap` stages the Android
 desktop stdlib (`GP/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt`, SPEC L-4).
 Old §15a, §15b, §15d, §15f, §15g, §15h are closed; §15e item 4 is closed by §15h.
 
-- **15.1 Nothing is published to a remote repository, open.** No publishing repository or signing is
-  configured (`signing { }` is commented out in `python-multiplatform/build.gradle.kts`), and
-  `repo1.maven.org/maven2/io/github/thisisthepy/python-multiplatform/` returns 404 (2026-10-03).
-  Every consumer verification so far used `mavenLocal()`. 10.1 depends on this or on a release asset.
+- **15.1 Maven Central publishing, implemented, first upload pending (#375).** All three components
+  (the library for every target, the KSP processor, the Gradle plugin with its plugin marker) carry
+  Central POM metadata, sources and javadoc jars and in-memory GPG signing
+  (`gradle/central-publishing.gradle.kts`, signing only when `MAVEN_GPG_PRIVATE_KEY` is set, so
+  `publishToMavenLocal` still works without keys). `./gradlew publishAllToCentralStaging` fills
+  `build/central-staging`; `.github/scripts/release/central-bundle.sh` signs, checks and zips it, and
+  `.github/workflows/maven-central.yml` (version tag, or manual with a `dry_run` input) builds on macOS and
+  uploads one deployment to the Central Portal (`central-upload.sh`, environment `maven-central`). The
+  dry run signs with a throwaway key and uploads nothing. Open: the first real upload (needs the
+  secrets, only CI can show Central accepting the deployment), and `repo1.maven.org/maven2/org/thisisthepy/`
+  does not list the artifacts until then. 10.1 depends on this or on a release asset.
 - **15.2 The library, KSP processor and plugin publish under one version, done (#70).** Each build
   derives it from `gradle.properties` (`$pythonVersion-$libraryVersionSuffix`, a `-P` override included);
   the root build fails when a project's version differs, and `PublishedVersionTest` pins the processor
diff --git a/gradle/central-publishing.gradle.kts b/gradle/central-publishing.gradle.kts
new file mode 100644
index 000000000..59b6db55d
--- /dev/null
+++ b/gradle/central-publishing.gradle.kts
@@ -0,0 +1,148 @@
+// Maven Central publishing, applied by the three published components (the library, the KSP
+// processor and the Gradle plugin, which is an included build):
+//
+//     apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))   // root build
+//     apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) // plugin build
+//
+// What it adds to every Maven publication of the project (ROADMAP 15.1):
+//   * the POM fields Central requires: name, description, url, license, developers, scm;
+//   * an empty javadoc jar and, where the publication has none, an empty sources jar (Central
+//     refuses a jar publication without both; pom-only plugin markers need neither);
+//   * a local `centralStaging` repository, a plain Maven layout under /build/central-staging,
+//     which `.github/scripts/release/central-bundle.sh` zips for the Central Portal upload API.
+//     Every component writes the same directory, so one deployment carries all of them;
+//   * in-memory PGP signing of every publication, only when a key is given
+//     (MAVEN_GPG_PRIVATE_KEY / MAVEN_GPG_PASSPHRASE in the environment, or the Gradle properties
+//     signingInMemoryKey / signingInMemoryKeyPassword). Without a key nothing is signed, so
+//     `publishToMavenLocal` keeps working on a machine that has none.
+// Nothing here uploads anything; the upload is .github/scripts/release/central-upload.sh.
+
+import org.gradle.api.publish.maven.MavenPublication
+import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven
+import org.gradle.api.publish.maven.tasks.PublishToMavenLocal
+import org.gradle.api.publish.maven.tasks.PublishToMavenRepository
+import org.gradle.plugins.signing.Sign
+import org.gradle.plugins.signing.SigningExtension
+
+val repoRoot: File = if (rootDir.resolve("gradle.properties").isFile && rootDir.resolve("gradle").isDirectory) {
+    rootDir
+} else {
+    rootDir.parentFile
+}
+val stagingDir: File = (findProperty("centralStagingDir") as String?)?.let(::File)
+    ?: repoRoot.resolve("build/central-staging")
+
+val signingKey: String? = (findProperty("signingInMemoryKey") as String?)
+    ?: System.getenv("MAVEN_GPG_PRIVATE_KEY")?.takeIf { it.isNotBlank() }
+val signingPassword: String = (findProperty("signingInMemoryKeyPassword") as String?)
+    ?: System.getenv("MAVEN_GPG_PASSPHRASE")
+    ?: ""
+
+val projectUrl = "https://github.com/thisisthepy/python-multiplatform"
+
+val emptyJar = { classifier: String ->
+    tasks.register("central${classifier.replaceFirstChar { it.uppercase() }}Jar") {
+        archiveClassifier.set(classifier)
+        destinationDirectory.set(layout.buildDirectory.dir("central-empty-jars"))
+        // Central only checks that the jar exists; the KDoc lives in the sources jar.
+        from(rootProject.file("LICENSE")) { into("META-INF") }
+    }
+}
+
+fun publicationIsBuilt(publication: org.gradle.api.publish.Publication, logger: org.gradle.api.logging.Logger): Boolean {
+    publication as MavenPublication
+    val missing = publication.artifacts.filter { !it.file.exists() }
+    if (missing.isNotEmpty()) {
+        logger.warn("CENTRAL-SKIPPED ${publication.groupId}:${publication.artifactId}: not built here: " +
+            missing.joinToString { it.file.name })
+    }
+    return missing.isEmpty()
+}
+
+pluginManager.withPlugin("maven-publish") {
+    apply(plugin = "signing")
+
+    val publishing = extensions.getByType()
+    publishing.repositories.maven {
+        name = "centralStaging"
+        url = stagingDir.toURI()
+    }
+
+    val javadocJar = emptyJar("javadoc")
+    val sourcesJar = emptyJar("sources")
+
+    publishing.publications.withType().configureEach {
+        val publication = this
+        // A plugin marker is a pom-only publication: nothing to attach.
+        val markerOnly = publication.name.endsWith("PluginMarkerMaven")
+        pom {
+            name.set(provider { publication.artifactId })
+            description.set(
+                "Embeds CPython in Kotlin Multiplatform and provides Kotlin <-> Python interoperability " +
+                    "(${publication.artifactId})."
+            )
+            url.set(projectUrl)
+            licenses {
+                license {
+                    name.set("The Apache License, Version 2.0")
+                    url.set("https://www.apache.org/licenses/LICENSE-2.0.txt")
+                }
+            }
+            developers {
+                developer {
+                    id.set("thisisthepy")
+                    name.set("thisisthepy")
+                    email.set("thisisthepy@gmail.com")
+                }
+            }
+            scm {
+                connection.set("scm:git:https://github.com/thisisthepy/python-multiplatform.git")
+                developerConnection.set("scm:git:ssh://git@github.com/thisisthepy/python-multiplatform.git")
+                url.set(projectUrl)
+            }
+        }
+        if (!markerOnly) {
+            // Attached once every plugin has added its own artifacts (KGP adds the sources jars of
+            // its targets late); a classifier already present is left alone.
+            gradle.projectsEvaluated {
+                val classifiers = publication.artifacts.mapNotNull { it.classifier }.toSet()
+                if ("javadoc" !in classifiers) publication.artifact(javadocJar)
+                if ("sources" !in classifiers) publication.artifact(sourcesJar)
+            }
+        }
+    }
+
+    if (signingKey != null) {
+        extensions.configure {
+            useInMemoryPgpKeys(signingKey, signingPassword)
+            sign(publishing.publications)
+        }
+        // KMP publications are created late and read the .asc files without declaring the tasks
+        // that write them; make every publish task wait for every signing task.
+        tasks.withType().configureEach {
+            dependsOn(tasks.withType())
+        }
+    }
+
+    // A publication whose file is not built on this machine is left out of the staging, loudly,
+    // rather than failing the whole deployment. Today that is only `wasmRuntime`: its zip needs a
+    // local Emscripten CPython build (-PwasmPythonDir) that CI does not make (ROADMAP 10.x), so
+    // `central-bundle.sh` lists what was left out and a release owner decides whether that is fine.
+    tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") }
+        .configureEach {
+            val task = this
+            onlyIf { publicationIsBuilt(task.publication, task.logger) }
+        }
+    // The same for mavenLocal, so `publishAllToMavenLocal` works in a checkout without the wasm build.
+    tasks.withType().configureEach {
+        val task = this
+        onlyIf { publicationIsBuilt(task.publication, task.logger) }
+    }
+
+    // One task per component that fills the staging directory with every publication.
+    tasks.register("publishAllToCentralStaging") {
+        group = "publishing"
+        description = "Publishes every publication of this project to $stagingDir (signed when a key is given)."
+        dependsOn(tasks.withType().matching { it.name.endsWith("ToCentralStagingRepository") })
+    }
+}
diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts
index 384ce7ce4..89fe990e6 100644
--- a/python-multiplatform-gradle-plugin/build.gradle.kts
+++ b/python-multiplatform-gradle-plugin/build.gradle.kts
@@ -418,3 +418,5 @@ tasks.test {
     inputs.files(walkerFixtureKlib).withPropertyName("walkerFixtureKlib").withPathSensitivity(PathSensitivity.NAME_ONLY)
     systemProperty("python.multiplatform.walkerFixtureKlib", walkerFixtureKlib.singleFile.absolutePath)
 }
+
+apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts"))
diff --git a/python-multiplatform-ksp/build.gradle.kts b/python-multiplatform-ksp/build.gradle.kts
index 67b3f22ac..c50576ae8 100644
--- a/python-multiplatform-ksp/build.gradle.kts
+++ b/python-multiplatform-ksp/build.gradle.kts
@@ -48,3 +48,5 @@ publishing {
         }
     }
 }
+
+apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))
diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts
index bb3ac4824..30a722feb 100644
--- a/python-multiplatform/build.gradle.kts
+++ b/python-multiplatform/build.gradle.kts
@@ -2789,47 +2789,11 @@ publishing {
             artifact(wasmBrowserRuntimeZip)
         }
     }
-    publications.withType().configureEach {
-        //artifact(javadocJar)
-        groupId = groupId
-        artifactId = artifactId
-        version = version
-
-        pom {
-            name.set("python-multiplatform")
-            description.set("A multiplatform solution to use Python with Kotlin interoperably.")
-            url.set("https://github.com/thisisthepy/python-multiplatform-mobile")
-            licenses {
-                license {
-                    name.set("The Apache License, Version 2.0")
-                    url.set("https://www.apache.org/licenses/LICENSE-2.0.txt")
-                }
-            }
-            developers {
-                developer {
-                    id.set("thisisthepy")
-                    name.set("thisisthepy")
-                    email.set("thisisthepy@gmail.com")
-                }
-            }
-            scm {
-                connection.set("scm:git:github.com/thisisthepy/python-multiplatform-mobile.git")
-                developerConnection.set("scm:git:ssh://github.com/thisisthepy/python-multiplatform-mobile.git")
-                url.set("https://github.com/thisisthepy/python-multiplatform-mobile")
-            }
-        }
-    }
 }
 
-//signing {
-//    useInMemoryPgpKeys(
-//        rootProject.extra["signing_key_id"] as String,
-//        rootProject.extra["signing_secret_key"] as String,
-//        rootProject.extra["signing_password"] as String
-//    )
-//    sign(publishing.publications)
-//}
-
+// POM metadata, the sources and javadoc jars, the centralStaging repository and the in-memory GPG
+// signing (only when MAVEN_GPG_PRIVATE_KEY is set) for Maven Central: ROADMAP 15.1.
+apply(from = rootProject.file("gradle/central-publishing.gradle.kts"))
 
 fun downloadPythonBuilds() {
     // Moved to task `downloadAllPythonBuilds`
diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
index a338dc59e..ec3d202f4 100644
--- a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
+++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/env/PackagedPythonHome.desktopNative.kt
@@ -1,12 +1,8 @@
 package python.multiplatform.env
 
 import kotlinx.cinterop.ExperimentalForeignApi
-import kotlinx.cinterop.alloc
-import kotlinx.cinterop.memScoped
-import kotlinx.cinterop.ptr
-import platform.posix.S_IFDIR
-import platform.posix.S_IFMT
-import platform.posix.stat
+import platform.posix.closedir
+import platform.posix.opendir
 
 /**
  * The directory containing the running executable, or null when the OS cannot report it.
@@ -67,12 +63,16 @@ internal object DesktopNativePythonHome {
         return null
     }
 
-    /** `stat(2)`'s answer: a directory exists at [path] (a regular file there is not one). */
+    /**
+     * A directory exists at [path] (a regular file there is not one): `opendir(3)` succeeds only on
+     * a directory. Not `stat(2)`: `st_mode` is 16 bits on macOS and 32 on Linux, and the metadata
+     * compilation of this shared source set, which publishing runs, rejects that mismatch.
+     */
     @OptIn(ExperimentalForeignApi::class)
-    internal fun isDirectory(path: String): Boolean = memScoped {
-        val info = alloc()
-        if (stat(path, info.ptr) != 0) return false
-        (info.st_mode.toInt() and S_IFMT) == S_IFDIR
+    internal fun isDirectory(path: String): Boolean {
+        val dir = opendir(path) ?: return false
+        closedir(dir)
+        return true
     }
 }
 

From 8fa5a33ddb8196b3759c4494565337b208d6835d Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:00:38 +0900
Subject: [PATCH 25/61] maven-central: install uv for the typedpython-gate
 bundle (#375)

---
 .github/workflows/maven-central.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml
index e79994d0b..d22eca2ee 100644
--- a/.github/workflows/maven-central.yml
+++ b/.github/workflows/maven-central.yml
@@ -72,6 +72,10 @@ jobs:
           java-version: '21'
           distribution: 'zulu'
 
+      - name: Setup uv
+        # The KSP processor's typedpython-gate bundle is built with uv build.
+        uses: astral-sh/setup-uv@v6
+
       - name: The tag names this build's version
         if: env.DRY_RUN != 'true'
         run: |

From cf909305f7d26be843230660186b31523bb2de17 Mon Sep 17 00:00:00 2001
From: BREW <81044362+b-re-w@users.noreply.github.com>
Date: Sat, 7 Nov 2026 11:24:06 +0900
Subject: [PATCH 26/61] Release: bump to 3.15.0-alpha03, the first Maven
 Central release (#377)

---
 docs/guide/getting-started.html                     | 4 ++--
 gradle.properties                                   | 2 +-
 python-multiplatform-gradle-plugin/build.gradle.kts | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html
index dd10ff835..47a73045c 100644
--- a/docs/guide/getting-started.html
+++ b/docs/guide/getting-started.html
@@ -77,11 +77,11 @@ 

04Use it from Maven Central

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

diff --git a/gradle.properties b/gradle.properties index 000eaca4e..069e2f372 100644 --- a/gradle.properties +++ b/gradle.properties @@ -34,7 +34,7 @@ pythonAppleSupportBuild=b11 # Every published artifact (library, KSP processor, Gradle plugin) is versioned # "$pythonVersion-$libraryVersionSuffix"; each build derives it from these two values and fails if # its own version disagrees (#70). -libraryVersionSuffix=alpha02 +libraryVersionSuffix=alpha03 # GraalVM native-image (sample's nativeCompile task). Liberica NIK is not in a location Gradle's # toolchain auto-detection scans, so it is registered explicitly here. Adding a path only makes a diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index 89fe990e6..5c20b0b6f 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -17,7 +17,7 @@ val rootGradleProperties = Properties().apply { } fun rootProperty(name: String, fallback: String): String = findProperty(name)?.toString() ?: rootGradleProperties.getProperty(name) ?: fallback -val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha02")}" +val publishedVersion = "${rootProperty("pythonVersion", "3.15.0")}-${rootProperty("libraryVersionSuffix", "alpha03")}" version = publishedVersion /** From ac5e0ddc1253680f12bdfee636ab2bb5205de651 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:55:13 +0900 Subject: [PATCH 27/61] Publish python-multiplatform-compose to Maven Central; drop stale Android libs; runnable getting-started (#379) - python-multiplatform-compose applies central-publishing (common, android, desktop) and joins publishAll* tasks. - copyAndroidPythonBinaries is a Sync; the link task removes stale libmultiplatform_python*.so; central-bundle.sh fails on an AAR that carries a libpython for another major.minor. - getting-started section 04: Kotlin version, desktop target, pythonBindings role app, upcall bootstrap, compose dependency; Central instead of mavenLocal; .tmp/ -> .cache/. --- .github/scripts/release/central-bundle.sh | 8 +++ build.gradle.kts | 15 +++--- docs/guide/getting-started.html | 50 ++++++++++++++++--- gradle/central-publishing.gradle.kts | 4 +- python-multiplatform-compose/build.gradle.kts | 9 ++++ python-multiplatform/build.gradle.kts | 13 ++++- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/.github/scripts/release/central-bundle.sh b/.github/scripts/release/central-bundle.sh index 55e8f8f2a..84e0b9585 100755 --- a/.github/scripts/release/central-bundle.sh +++ b/.github/scripts/release/central-bundle.sh @@ -48,6 +48,14 @@ while IFS= read -r pom; do [[ -e "$dir/$base-sources.jar" ]] || { echo "error: $art has no sources jar" >&2; fail=1; } fi done < <(find "$staging" -name '*.pom') +# An AAR must carry the native libraries of this pythonVersion only. A Copy-staged jniLibs tree once +# kept libpython3.14.so beside libpython3.15.so after a version bump. libpython3.so (the stable-ABI +# stub) carries no minor version and is allowed. +py_mm="$(sed -nE 's/^pythonVersion=([0-9]+\.[0-9]+).*$/\1/p' gradle.properties)" +while IFS= read -r aar; do + stale="$(unzip -Z1 "$aar" | grep -E '(^|/)lib(multiplatform_)?python[0-9]+\.[0-9]+[^/]*\.so$' | grep -vE "(^|/)lib(multiplatform_)?python${py_mm//./\\.}(t)?\.so$" || true)" + [[ -z "$stale" ]] || { echo "error: $aar carries native libraries for another python than $py_mm:" >&2; echo "$stale" >&2; fail=1; } +done < <(find "$staging" -name '*.aar') if $signed; then while IFS= read -r f; do case "$f" in *.asc|*.md5|*.sha1|*.sha256|*.sha512) continue ;; esac diff --git a/build.gradle.kts b/build.gradle.kts index cad457f0d..c1a07f833 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -34,27 +34,30 @@ gradle.projectsEvaluated { // (see `pluginManagement { includeBuild(...) }` in settings.gradle.kts), so its tasks never join // the root task graph -- `./gradlew tasks --all` from here lists none of them, and a coordinator // publishing only what the root build shows would silently skip it. This task does not change how -// any of the three components publish (each already has its own `maven-publish` config); it only -// gives the root build one entry point that reaches all three. +// any component publishes (each already has its own `maven-publish` config); it only gives the root +// build one entry point that reaches all of them (the Compose host included). tasks.register("publishAllToMavenLocal") { group = "publishing" - description = "Publishes python-multiplatform, python-multiplatform-ksp, and " + + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + "python-multiplatform-gradle-plugin (including its plugin-marker publication) to mavenLocal()." dependsOn(":python-multiplatform:publishToMavenLocal") dependsOn(":python-multiplatform-ksp:publishToMavenLocal") + dependsOn(":python-multiplatform-compose:publishToMavenLocal") dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) } -// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with all three -// components. Signed when MAVEN_GPG_PRIVATE_KEY is set (gradle/central-publishing.gradle.kts). +// Maven Central (ROADMAP 15.1): one entry point that fills build/central-staging with every +// component (library, KSP processor, Compose host, Gradle plugin). Signed when MAVEN_GPG_PRIVATE_KEY +// is set (gradle/central-publishing.gradle.kts). // It does not clean the directory (the plugin build is another build, so ordering cannot reach // it): .github/scripts/release/central-bundle.sh empties it first. tasks.register("publishAllToCentralStaging") { group = "publishing" - description = "Publishes python-multiplatform, python-multiplatform-ksp and " + + description = "Publishes python-multiplatform, python-multiplatform-ksp, python-multiplatform-compose and " + "python-multiplatform-gradle-plugin (with its plugin marker) to build/central-staging." dependsOn(":python-multiplatform:publishAllToCentralStaging") dependsOn(":python-multiplatform-ksp:publishAllToCentralStaging") + dependsOn(":python-multiplatform-compose:publishAllToCentralStaging") dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishAllToCentralStaging")) } diff --git a/docs/guide/getting-started.html b/docs/guide/getting-started.html index 47a73045c..976e6e60b 100644 --- a/docs/guide/getting-started.html +++ b/docs/guide/getting-started.html @@ -34,7 +34,7 @@
Getting started시작하기

From clone to a running sample클론부터 샘플 실행까지

-

No release is published yet, so you build from source. The sample app is the shortest path to seeing both directions work.아직 배포된 릴리스가 없으므로 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

+

Releases are on Maven Central (section 04). To work on the library itself, build from source: the sample app is the shortest path to seeing both directions work.릴리스는 Maven Central 에 있다(04 절). 라이브러리 자체를 고치려면 소스에서 빌드한다. 두 방향이 동작하는 것을 보는 가장 짧은 길은 샘플 앱이다.

01Requirements준비물

@@ -70,25 +70,61 @@

03Your first call

There is no close() in that code, and none is missing: every wrapper releases its Python reference when the Kotlin object is collected.위 코드에는 close() 가 없고, 빠진 것도 아니다. 모든 래퍼는 Kotlin 객체가 수거될 때 Python 참조를 놓는다.

04Use it from Maven CentralMaven Central 에서 쓰기

+

Consumers must build with Kotlin 2.4.20-Beta2, the version the artifacts are compiled with. A desktop app that Python calls back into:소비자는 아티팩트가 컴파일된 Kotlin 2.4.20-Beta2 로 빌드해야 한다. Python 이 다시 호출하는 데스크톱 앱:

// settings.gradle.kts
 pluginManagement { repositories { mavenCentral(); gradlePluginPortal() } }
-dependencyResolutionManagement { repositories { mavenCentral() } }
+dependencyResolutionManagement { repositories { mavenCentral(); google() } }
 
-// build.gradle.kts of a Kotlin Multiplatform module
+// build.gradle.kts of a Kotlin Multiplatform app
 plugins {
-    kotlin("multiplatform")
+    kotlin("multiplatform") version "2.4.20-Beta2"
     id("org.thisisthepy.python.multiplatform.bindings") version "3.15.0-alpha03"
 }
 kotlin {
+    jvm("desktop") {
+        @OptIn(org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi::class)
+        binaries { executable { mainClass.set("app.MainKt") } }
+    }
     sourceSets.commonMain.dependencies {
         implementation("org.thisisthepy:python-multiplatform:3.15.0-alpha03")
+        // Compose hosts (PythonAppView, PythonWidget, PythonLauncher):
+        // implementation("org.thisisthepy:python-multiplatform-compose:3.15.0-alpha03")
     }
+}
+// A KMP executable is otherwise treated as a library and gets no FunctionTable.
+pythonBindings { role.set("app") }
+
// src/desktopMain/kotlin/app/Main.kt
+package app
+
+import python.multiplatform.ffi.Python3
+import python.multiplatform.ffi.upcall.PythonProxySource
+import python.native.ffi.UpcallStub
+
+class Greeter(val subject: String) {
+    fun greet(times: Long): String = List(times.toInt()) { "hello $subject" }.joinToString(", ")
+}
+
+fun main() {
+    Python3.initialize()
+    // Upcalls: install the generated table, bind the entry points, install the proxies.
+    python.multiplatform.generated.FunctionTable.installInto()
+    Python3.exec(
+        """
+        import ctypes
+        _pm_resolve = ctypes.CFUNCTYPE(ctypes.c_long, ctypes.c_char_p)(${UpcallStub.resolveHandleStubAddr})
+        _pm_invoke = ctypes.CFUNCTYPE(ctypes.py_object, ctypes.c_long, ctypes.py_object)(${UpcallStub.invokeWithArgsStubAddr})
+        _pm_release = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.releaseObjectStubAddr})
+        _pm_cancel = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_long)(${UpcallStub.cancelCallStubAddr})
+        """.trimIndent(),
+    )
+    PythonProxySource.install()
+    Python3.exec("from app import Greeter\nprint(Greeter('Kotlin').greet(2), flush=True)")
 }
-

The library, the KSP processor and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix. A version tag publishes them in one deployment through .github/workflows/maven-central.yml; until a release has been published there, resolve from mavenLocal() after ./gradlew publishAllToMavenLocal.라이브러리, KSP 프로세서, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓴다. 버전 태그를 달면 .github/workflows/maven-central.yml 이 셋을 한 번의 배포로 올린다. 릴리스가 올라가기 전에는 ./gradlew publishAllToMavenLocal 뒤에 mavenLocal() 에서 가져온다.

+

The library, the KSP processor, the Compose host and the Gradle plugin share one version, $pythonVersion-$libraryVersionSuffix, and are published together on Maven Central; the current release is 3.15.0-alpha03. google() is needed for Android targets.라이브러리, KSP 프로세서, Compose 호스트, Gradle 플러그인은 $pythonVersion-$libraryVersionSuffix 라는 한 버전을 쓰고 Maven Central 에 함께 올라간다. 현재 릴리스는 3.15.0-alpha03 이다. Android 타깃에는 google() 이 필요하다.

05Verify a change변경 검증하기

-
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .tmp/build.log 2>&1; echo "EXIT=$?"
-./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .tmp/test.log 2>&1; echo "EXIT=$?"
+  
./gradlew :python-multiplatform:compileKotlinAndroidNativeArm64 --console=plain > .cache/build.log 2>&1; echo "EXIT=$?"
+./gradlew :python-multiplatform:desktopTest --rerun --console=plain > .cache/test.log 2>&1; echo "EXIT=$?"
 ./gradlew :python-multiplatform-ksp:fixtures:app:desktopTest --rerun
 ./gradlew :python-multiplatform-compose:fixtures:compose:desktopTest --rerun
 ./gradlew :python-multiplatform-gradle-plugin-fixtures:artifact:desktopTest --rerun
diff --git a/gradle/central-publishing.gradle.kts b/gradle/central-publishing.gradle.kts index 59b6db55d..c108f2bb9 100644 --- a/gradle/central-publishing.gradle.kts +++ b/gradle/central-publishing.gradle.kts @@ -1,5 +1,5 @@ -// Maven Central publishing, applied by the three published components (the library, the KSP -// processor and the Gradle plugin, which is an included build): +// Maven Central publishing, applied by the published components (the library, the KSP processor, +// the Compose host and the Gradle plugin, which is an included build): // // apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) // root build // apply(from = rootDir.resolve("../gradle/central-publishing.gradle.kts")) // plugin build diff --git a/python-multiplatform-compose/build.gradle.kts b/python-multiplatform-compose/build.gradle.kts index 011f95798..353384d43 100644 --- a/python-multiplatform-compose/build.gradle.kts +++ b/python-multiplatform-compose/build.gradle.kts @@ -15,10 +15,14 @@ plugins { alias(libs.plugins.android.library) alias(libs.plugins.jetpack.compose) alias(libs.plugins.compose.compiler) + id("maven-publish") } kotlin { androidTarget { + // Without a named variant KMP creates no Android publication (ROADMAP 15d, same as the library). + publishLibraryVariants("release") + @OptIn(ExperimentalKotlinGradlePluginApi::class) compilerOptions { jvmTarget.set(JvmTarget.JVM_17) @@ -52,3 +56,8 @@ android { targetCompatibility = JavaVersion.VERSION_17 } } + +// Published to Maven Central beside the library as org.thisisthepy:python-multiplatform-compose, at the +// root's one version (#70): POM fields, sources/javadoc jars, centralStaging repository and in-memory +// signing come from the shared script (ROADMAP 15.1). +apply(from = rootProject.file("gradle/central-publishing.gradle.kts")) diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index 30a722feb..e9f99602c 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -1067,7 +1067,11 @@ kotlin { } afterEvaluate { val abiList = listOf("arm64-v8a", "x86_64") - val copyAndroidPythonBinaries by tasks.creating(Copy::class) { + // A Sync, not a Copy: a Copy never removes what it stopped copying, so a jniLibs tree + // staged under an older pythonVersion (libpython3.14.so beside libpython3.15.so) kept + // shipping in the AAR. Nothing else writes this directory (the link task stages into + // `$androidBuildDir//jniLibs`), so the Sync owns all of it. + val copyAndroidPythonBinaries by tasks.creating(Sync::class) { dependsOn( tasks.named("linkAndroidNativeArm64"), tasks.named("linkAndroidNativeX64"), @@ -1301,7 +1305,14 @@ kotlin { // outputFile. That is why a changed .def or source produced an APK with // a stale library and an UnsatisfiedLinkError that looked like a code // bug -- the fix had to be built twice for the second run to pick it up. + // A library linked under an older pythonVersion + // (libmultiplatform_python3.14.so) is removed first, or it keeps + // shipping in the AAR beside the current one. doLast { + val current = outputFile.get().name + dest.listFiles { f -> + f.name.startsWith("libmultiplatform_python") && f.name.endsWith(".so") && f.name != current + }?.forEach { it.delete() } copy { from(outputFile) into(dest) From aade0f8d162bacae09e505d38283d74999cff819 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:25:55 +0900 Subject: [PATCH 28/61] AGENTS 7: what counts as the owner's decision (shared rule) --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 105f61706..8dda6210b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,27 @@ never force-push without the user's explicit approval. When a rule and backward compatibility conflict, **the rule wins.** List the callers that break and fix them; do not keep the forbidden thing "so nothing breaks". +### What counts as the owner's decision + +Added after an agent turned the owner's question "should ExecuTorch be the NPU back end?" into an +offline per-model `.pte` pipeline with its own CI, recorded it as "decided, do not re-discuss" and +later as project intent. The owner had approved none of it (torchnative, 2026-08 to 2026-11). + +- **A question is not a decision.** "X를 쓰는게 좋을까?", "should we use X?" opens a decision. If the + owner did not answer, it stays open; never record it as settled. +- **Approving a goal is not approving a means.** "Make the Snapdragon NPU work, go ahead" approves + the goal. The engine, the dependencies, the pipeline, the CI workflow and the artefact types an + agent picks to reach it are separate decisions. +- **Never restate the owner's question as a different one.** When asking for a decision, quote the + owner's words and the constraint behind them (for example "NNAPI is deprecated, so what + abstraction reaches every vendor's NPU?"), and give options that answer that question. +- **Ask before adding** a dependency, a pipeline, a CI workflow or a kind of artefact the owner did + not name. Show the options and wait. +- **Docs keep provenance.** A line in INTENT, SPEC, AGENTS or a design doc that states a direction + cites the owner's words (date and source). Anything else is marked "agent proposal, not + confirmed". Never write "decided", "settled" or "do not re-discuss" for a choice the owner did not + make, and keep that mark when a doc is restructured. + ## 8. Verification that can fail - Never read a build's exit code through a pipe (`| tail`, `| grep`). Redirect to a file, then read From 9e0718b6801f99c494357dd8fb903bc6a73ecb0d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:36:24 +0900 Subject: [PATCH 29/61] AGENTS 3, 10: clean up on merge; agents only check compilation --- AGENTS.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8dda6210b..7b7791293 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,8 +50,14 @@ build caches, model weights, `node_modules`) into every worktree is how 86 workt - Create worktrees under `.worktrees/`. - **Symlink** large untracked directories from the main checkout instead of copying or rebuilding them. If `tools/worktree-add.sh` exists, use it, it does the linking. -- Delete a worktree once its branch is merged: `git worktree remove .worktrees/`. -- Periodically delete `build/` directories inside worktrees; they only grow. +- **When a branch merges, clean up in the same step**: remove its worktree + (`git worktree remove .worktrees/`), delete the local and remote branch, and delete every + build output, cache or target directory created for that work (per-branch `build/`, a + `CARGO_TARGET_DIR`, a venv, a scratch directory). Work is not finished while any of these remain. + This was added after 32 per-agent cargo target directories (33 GB) were left behind in one + repository. +- Never create a per-agent or per-branch build cache (a separate `CARGO_TARGET_DIR`, a copied + `.gradle`, a private venv). Use the repository's one canonical location. ## 4. Branches @@ -181,10 +187,13 @@ to implement", say what you counted against. counts, lifetimes, class loaders) gets the strongest tier; work a test will catch can use a cheaper one. - Give every agent prompt the absolute paths it may write to, and repeat rule 2 in it. -- **Subagents do not run heavy local builds.** Subagents write code, design, investigate, review - and document. Gradle builds, cargo builds, the test gate and model runs are done by the session - itself, one at a time on this machine, or by CI (GitHub Actions) on a pushed branch. Several - sessions share one machine; parallel local builds slow every one of them. +- **Agents do not build.** Every agent, teammate sessions and subagents alike, may check that code + compiles and nothing more: `cargo check`, a Gradle `compile*` task, `python -m py_compile`, a + type checker or linter. Full builds (`cargo build`, `assemble`, wheels, native images, app + packages), test suites, gates, benchmarks and model runs go to CI (GitHub Actions) on a pushed + branch. The only local exceptions are device tests and measurements the coordinating session + runs itself, one at a time, and says why. Several sessions share one machine, and agent builds + once filled the disk and slowed every session. --- From 6eae2c43a7dd22ac6747bec49ef33aa5b0893629 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:26:05 +0900 Subject: [PATCH 30/61] TypedPython: record benchmark output identity in CI and index the differential tests (#41) test_benchmark_outputs.py compiles the five benchmarks and compares their output with the interpreter at the run.py verify sizes, inside the TypedPython CI. docs/design/typedpython.md 4.3.1 lists the differential tests per edge case named by the M2 criterion. --- docs/design/typedpython.md | 10 ++ .../src/test/python/test_benchmark_outputs.py | 91 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index aee22b5d6..010427201 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -155,6 +155,16 @@ API 안정성을 확인한 뒤로 미룬다. 않고** 인터프리터에 남기며 이유를 진단으로 낸다. 생성 C 는 `tp_runtime.h` 헬퍼로만 메모리와 `PyObject` 를 다룬다. 증명 밖의 안전망: ASan/UBSan, 인터프리터와의 차등 테스트, IR 수준 퍼징. +차등 테스트의 위치 (#41 M2 기준, 경로는 `python-multiplatform-ksp/src/test/python/`): + +| 경우 | 테스트 | +|---|---| +| i64 오버플로 경계 | `test_runtime.py::test_checked_arithmetic_matches_bigint_and_deopts_exactly_on_overflow`, `test_neg_i64_deopts_only_for_int64_min`, `test_fuzz_checked_arithmetic_against_bigint`; 컴파일된 함수에서 `test_cgen.py::test_i64_overflow_in_a_pure_function_deopts_to_the_big_int_result`, `test_native_int_impure.py::test_binary_ops_at_the_i64_boundary_equal_cpython`, `test_min_floordiv_minus_one_promotes` | +| 음수 floor 나눗셈과 나머지 | `test_runtime.py::test_floordiv_i64_is_python_floor_division`, `test_mod_i64_has_the_sign_of_the_divisor`, `test_fuzz_floordiv_mod_i64`, `test_floor_mod_sign_and_zero_cases_spelled_out`, `test_fuzz_float_floordiv_mod` | +| 0 으로 나누기 | `test_runtime.py::test_int_division_by_zero_messages_are_cpythons`, `test_float_division_by_signed_zero_raises_cpythons_message`; `test_cgen.py::test_integer_division_by_zero_has_cpythons_message`, `test_native_int_impure.py::test_division_by_zero_after_a_side_effect` | +| NaN 과 inf | `test_runtime.py::test_float_division_family_is_bit_exact_with_cpython`, `test_int_float_comparison_named_hard_cases`, `test_math_named_error_cases`, `test_math_one_argument_fuzz_is_bit_exact` (NaN 은 "어떤 NaN", 그 밖은 IEEE 비트 비교); `test_pipeline.py::test_fastmath_keeps_nan_checks` | +| 벤치마크 출력 | `test_benchmark_outputs.py`: 다섯 벤치마크를 `run.py verify` 크기로 컴파일 실행해 인터프리터 출력과 비교, TypedPython CI 가 develop 의 매 push 마다 돌린다 | + ### 4.3.2 증분 컴파일 (SPEC N-10) 사용자 결정: "증분 컴파일도 지원하도록 해. 빌드 속도 너무 오래 안걸리게". 단위는 모듈. 키는 소스 해시 + diff --git a/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py b/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py new file mode 100644 index 000000000..b433d1554 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_benchmark_outputs.py @@ -0,0 +1,91 @@ +"""TypedPython: the five benchmarks print exactly what the interpreter prints (#41, M2 criterion). + +The M2 criterion of #41 asks for a record that the compiled benchmarks give output identical to +CPython. `benchmarks/typedpython/run.py verify --typedpython` checks that too, but it also needs +Node, Java and Rust, so it does not run in the TypedPython CI. This file runs the same comparison +inside that CI, on every push to develop, so the record carries the commit and date of each run. + +For each benchmark: the source with `@compiled` above every top-level def and class is compiled +through `pipeline.compile_module`; its `main()` runs at the size `run.py verify` uses, and its stdout +is compared with the plain source run by the interpreter in the same process. The compiled-function +list is printed (`pytest -s` shows it) so a run also records what was compiled; the list itself is +pinned by `test_class_methods.py::test_the_benchmarks_compile_the_same_functions_as_before`. + +What this cannot find: a difference that only shows at the benchmark size, or only under concurrent +callers. The differential tests listed in docs/design/typedpython.md cover the arithmetic edge cases. + +Written after the implementation (a record of existing behaviour, not a red/green test): it passes on +develop today, and a mismatch is a regression. +""" +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +from pathlib import Path + +import pytest + +from conftest import decorate_compiled +from typedpython import pipeline + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +BENCH = REPO / "benchmarks" / "typedpython" / "py" + +# name -> (source file, size), the verify sizes of benchmarks/typedpython/run.py BENCHMARKS +SIZES = { + "nbody": ("nbody.py", "5000"), + "spectral-norm": ("spectral_norm.py", "100"), + "fannkuch-redux": ("fannkuch.py", "7"), + "binary-trees": ("binary_trees.py", "10"), + "wordfreq": ("wordfreq.py", "100000"), +} + + +def run_main(module, name: str, size: str) -> str: + old = sys.argv + sys.argv = [name, size] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + module.main() + finally: + sys.argv = old + return buf.getvalue() + + +def load_plain(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("name", sorted(SIZES)) +def test_compiled_benchmark_output_equals_the_interpreter(tmp_path, name): + py, size = SIZES[name] + source = (BENCH / py).read_text() + stem = py.removesuffix(".py") + + plain_path = tmp_path / f"{stem}_ref.py" + plain_path.write_text(source) + expected = run_main(load_plain(plain_path, f"{stem}_ref"), name, size) + assert expected.strip(), f"{name}: the interpreted run printed nothing" + + path = tmp_path / py + path.write_text(decorate_compiled(source)) + result = pipeline.compile_module(path, tmp_path / "out") + compiled = sorted(f.name for f in result.module.functions) + print(f"\n{name}: compiled {compiled}, skipped {sorted(result.skipped)}") + if result.extension is None: + # nothing compiled (nbody on free-threaded builds until #255): the comparison would be + # interpreter against interpreter, so it is reported, not counted as a pass + assert compiled == [] + pytest.skip(f"{name}: no function compiled on this build, nothing to compare") + + module = pipeline.load(result) + before = getattr(module, "__typedpython_deopts__", None) + assert run_main(module, name, size) == expected + print(f"{name}: output identical, deopts {before} -> {getattr(module, '__typedpython_deopts__', None)}") From 4c714cb6ec96e854af3df3b58ea2b6aa4a10d08c Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:09:15 +0900 Subject: [PATCH 31/61] Plugin: iOS consumer wiring can stage the project's free-threaded 3.15t xcframework (#328 step 1) Downloads python-3.15.0t-iOS-XCframework.tar.gz from release cpython-3.15.0t-mobile, verified against the pinned SHA-256 ios-3.15.0-freethreaded, and stages lib/python3.15t per slice. Selected with pythonBindings.ios.freeThreaded (agent proposal, default false). Android pin recorded for step 2. SPEC T-2a (partial). --- docs/SPEC.md | 23 ++++- python-checksums.properties | 2 + .../build.gradle.kts | 18 ++++ .../multiplatform/gradle/IosConsumerWiring.kt | 63 ++++++++++++- .../multiplatform/gradle/IosPythonHome.kt | 32 +++++-- .../gradle/IosConsumerWiringTest.kt | 67 ++++++++++++++ .../gradle/IosPythonHomeLayoutTest.kt | 91 +++++++++++++++++++ 7 files changed, 281 insertions(+), 15 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 90d07e494..537b64f3a 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -658,8 +658,27 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. asset. `Status: partial` (desktop only, opt-in; default is `pythonFreeThreaded=false` in `gradle.properties`) , `GP/PythonHomeStagingTest.kt`, `PM/desktopTest/.../FreeThreadedGCGateTest.kt`, `VersionsTest.kt`. 3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (236 tests, 0 failures; ROADMAP §9). - Only desktop has free-threaded prebuilts; Android and iOS do not. `Py_LIMITED_API` is not defined, so - `abi3t` is not a blocker. + python.org and python-build-standalone publish free-threaded prebuilts for desktop only; for Android + and iOS this project builds its own (#163, release `cpython-3.15.0t-mobile`), see T-2a. + `Py_LIMITED_API` is not defined, so `abi3t` is not a blocker. +- **T-2a** Free-threaded CPython on mobile through the Gradle plugin (#328 step 1; direction: the + maintainer's #158 decision of 2026-10-04, "Free-threaded is the default and the only build", baseline + 3.15t). The plugin's iOS consumer wiring (#90, L-11) can acquire the project's own free-threaded + `Python.xcframework`, `python-t-iOS-XCframework.tar.gz` from the GitHub release + `cpython-t-mobile` of `thisisthepy/python-multiplatform`, verified against the SHA-256 pinned in + `python-checksums.properties` under `ios--freethreaded` and baked into the plugin at plugin build + time (a mismatch, or a missing pin, fails `downloadPython_ios`). It is selected per consumer with + `pythonBindings { ios { freeThreaded.set(true) } }`; the default stays `false`, the python.org GIL + archive and `lib/python/`, until #328 step 3 switches the mobile default (the separate switch, + and its default, are an agent proposal, not confirmed). With it set, every per-slice prefix holds + `lib/pythont/`: the shared `Python.xcframework/lib/pythont/` merged with + `/lib-/pythont/`, with L-11's exclusions (so no `libpythont.dylib`), and the + staging task fails unless `lib/pythont/os.py` is staged. Android: the plugin stages nothing for + Android (libpython and the stdlib travel inside the library's AAR, L-5); the Android archive's pin + (`android-aarch64--freethreaded`) is recorded in `python-checksums.properties`, and acquiring it + is a library-build change left to #328 steps 2 and 3. `Status: partial` (unit tests only, + `GP/IosConsumerWiringTest.kt`, `GP/IosPythonHomeLayoutTest.kt`; no simulator run, and the library's own + iOS klib is still compiled against the GIL headers). - **T-3** `python-multiplatform` publishes per-target, per-flavour CPython include directories as `Provider` carrying the extraction task (`cpythonIncludeDirectories` extension; see `docs/platforms/python-version-acquisition.md` §7). `Status: partial` (only the configured flavour is diff --git a/python-checksums.properties b/python-checksums.properties index 806f0bf69..b3899bd58 100644 --- a/python-checksums.properties +++ b/python-checksums.properties @@ -1,9 +1,11 @@ # Python Multiplatform Checksums android-aarch64-3.14.7=6d50cc3aa66e414a439594089bcdfb5f1264358155c70c1f00471c24cfb477fb android-aarch64-3.15.0=571e4cf4e0aee3d1ab54a52ca3d4e6845b85ec0bb327f7debfb873172f445dab +android-aarch64-3.15.0-freethreaded=40995c530c3e454d829b4fdea8404d8c0b0e58f60c2d41d400daa1fb0ab65991 android-x86_64-3.14.7=2c16ce2359565cd8c24f86cfb75630768ba6607e732946b294b969797f583b60 android-x86_64-3.15.0=2413d51948d612cf42baefba25295bb83c2ed84df5726af6fe5bde2dec5d87e7 ios-3.14-b11=b591f3301bd22a4f423c49c746cac9e55558b909fd14d6eb8327ccc62234ab7b +ios-3.15.0-freethreaded=b8c0a4b32d31738f6ead7ace28acee0cb2a590bd0ca7015e9c32af613445e6b5 ios-3.15.0-pythonorg=68e98ab35cc0643ba3a227a8c7768da45c2a5e67dbe1bc26598537e96680fd19 linux-x86_64-3.14.7-20260807=a2478d654ed51d443bae21ec20ad927f116b4f5aae4094ab74918a6aa38f0575 linux-x86_64-3.14.7-20260807-freethreaded=13f80b7fc242814c0be4e2a4d8283f39c7e1c0f3f0d8f6d6a15cedb5b62eb6d3 diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index 5c20b0b6f..f7229c7f0 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -289,6 +289,16 @@ val generateCoordinates = tasks.register("generateCoordinates") { "python-checksums.properties (${checksumsFile.path}) has no '$iosLockKeyValue'; the plugin bakes the iOS support archive's SHA-256 from it.", ) + // The project's own free-threaded iOS build (#163, #328 step 1): release `cpython-t-mobile` + // of thisisthepy/python-multiplatform. Pinned under `ios--freethreaded`; the 3.15.0 value + // is the release's SHA256SUMS line, equal to the GitHub asset digest and to `shasum -a 256` of the + // downloaded archive (checked 2026-11-13). Optional: a missing pin bakes "" and only a consumer + // that selects the free-threaded archive fails, in `downloadPython_ios`, naming the key. + val iosFreeThreadedLockKeyValue = "ios-$pythonVersionValue-freethreaded" + val iosFreeThreadedSha256Value = Properties().apply { + if (checksumsFile.isFile) checksumsFile.inputStream().use { load(it) } + }.getProperty(iosFreeThreadedLockKeyValue).orEmpty() + inputs.property("coordinates", coordinates) inputs.property("klibWorkerCoordinates", klibWorkerValue) inputs.property("pythonVersion", pythonVersionValue) @@ -299,6 +309,8 @@ val generateCoordinates = tasks.register("generateCoordinates") { inputs.property("iosArchiveName", iosArchiveNameValue) inputs.property("iosLockKey", iosLockKeyValue) inputs.property("iosSha256", iosSha256Value) + inputs.property("iosFreeThreadedLockKey", iosFreeThreadedLockKeyValue) + inputs.property("iosFreeThreadedSha256", iosFreeThreadedSha256Value) outputs.dir(outputDir) doLast { val file = outputDir.get().file("python/multiplatform/gradle/ProcessorCoordinates.kt").asFile @@ -328,6 +340,12 @@ val generateCoordinates = tasks.register("generateCoordinates") { /** Its SHA-256, read from `python-checksums.properties` when this plugin was built. */ internal const val DEFAULT_IOS_ARCHIVE_SHA256: String = "$iosSha256Value" + /** The key of the free-threaded iOS archive (SPEC T-2a) in `python-checksums.properties`. */ + internal const val DEFAULT_IOS_FREE_THREADED_LOCK_KEY: String = "$iosFreeThreadedLockKeyValue" + + /** Its SHA-256, or "" when `python-checksums.properties` had no such key at plugin build time. */ + internal const val DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256: String = "$iosFreeThreadedSha256Value" + internal const val DEFAULT_PYTHON_FREE_THREADED: Boolean = $freeThreadedValue /** diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt index ae7f178f6..ed615f1c0 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt @@ -33,8 +33,55 @@ interface PythonBindingsIosExtension { * `Python.framework` some other way. */ val linkPythonFramework: Property + + /** + * Whether to acquire the free-threaded CPython (`3.15t`) `Python.xcframework` this project builds + * and publishes on its own GitHub release, instead of python.org's GIL one (SPEC T-2a, #328 step 1). + * Defaults to false until #328 step 3 switches the mobile default. + * + * Separate from `pythonBindings.pythonFreeThreaded` on purpose: that one follows the desktop + * flavour the library was built with, while the library's own iOS klib is still compiled against + * the GIL headers. Deriving iOS from it would pair a GIL klib with a free-threaded framework the + * moment the library is built with `-PpythonFreeThreaded=true`. Agent proposal, not confirmed. + */ + val freeThreaded: Property +} + +/** + * The project's own free-threaded CPython builds for Android and iOS (#163, `build-ft-mobile.yml`), + * published on the GitHub release `cpython-t-mobile` of `thisisthepy/python-multiplatform` + * because neither python.org nor BeeWare publishes a free-threaded mobile build. Names only; the + * SHA-256 pins live in the root `python-checksums.properties` under [lockKey]. + */ +internal object FreeThreadedMobileRelease { + const val REPOSITORY: String = "thisisthepy/python-multiplatform" + + fun tag(pythonVersion: String): String = "cpython-${pythonVersion}t-mobile" + + fun iosArchiveName(pythonVersion: String): String = "python-${pythonVersion}t-iOS-XCframework.tar.gz" + + /** `aarch64` is the only Android architecture the release carries (no x86_64 build exists). */ + fun androidArchiveName(pythonVersion: String, arch: String): String = "python-${pythonVersion}t-$arch-linux-android.tar.gz" + + fun url(pythonVersion: String, assetName: String): String = + "https://github.com/$REPOSITORY/releases/download/${tag(pythonVersion)}/$assetName" + + /** `ios-3.15.0-freethreaded`, `android-aarch64-3.15.0-freethreaded`. */ + fun lockKey(platform: String, pythonVersion: String): String = "$platform-$pythonVersion-freethreaded" } +/** One iOS support archive the consumer wiring can download: where from, under which name, which pin. */ +internal data class IosArchive(val url: String, val name: String, val sha256: String) + +/** The archive [configureIosConsumer] downloads for the selected flavour of [DEFAULT_PYTHON_VERSION]. */ +internal fun iosArchiveFor(freeThreaded: Boolean): IosArchive = + if (freeThreaded) { + val name = FreeThreadedMobileRelease.iosArchiveName(DEFAULT_PYTHON_VERSION) + IosArchive(FreeThreadedMobileRelease.url(DEFAULT_PYTHON_VERSION, name), name, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) + } else { + IosArchive(DEFAULT_IOS_ARCHIVE_URL, DEFAULT_IOS_ARCHIVE_NAME, DEFAULT_IOS_ARCHIVE_SHA256) + } + internal const val DOWNLOAD_IOS_TASK = "downloadPython_ios" internal const val PREPARE_XCFRAMEWORK_TASK = "preparePythonXcframework" internal const val STAGE_IOS_HOME_TASK = "stageIosPythonHome" @@ -106,6 +153,13 @@ abstract class DownloadPythonIosTask : DefaultTask() { work.mkdirs() val archive = File(work, archiveName.get()) val expected = expectedSha256.get().lowercase() + if (expected.isBlank()) { + throw GradleException( + "There is no pinned SHA-256 for ${archiveName.get()}: the root python-checksums.properties had no " + + "entry for it (free-threaded: '$DEFAULT_IOS_FREE_THREADED_LOCK_KEY') when this plugin was built. " + + "Nothing was downloaded.", + ) + } if (archive.isFile && sha256Hex(archive) != expected) { logger.lifecycle("The cached ${archive.name} does not match its pinned SHA-256; deleting it and downloading once more.") @@ -202,13 +256,15 @@ internal fun PythonBindingsPlugin.configureIosConsumer(project: Project, extensi if (iosTargets.isEmpty()) return val build = project.layout.buildDirectory + val freeThreaded = extension.ios.freeThreaded.orElse(false) + val archive = freeThreaded.map { iosArchiveFor(it) } val download = project.tasks.register(DOWNLOAD_IOS_TASK, DownloadPythonIosTask::class.java) { group = "python" description = "Downloads and verifies the pinned CPython iOS support archive into build/python-standalone (issue #90)" - archiveUrl.convention(DEFAULT_IOS_ARCHIVE_URL) - archiveName.convention(DEFAULT_IOS_ARCHIVE_NAME) - expectedSha256.convention(DEFAULT_IOS_ARCHIVE_SHA256) + archiveUrl.convention(archive.map { it.url }) + archiveName.convention(archive.map { it.name }) + expectedSha256.convention(archive.map { it.sha256 }) workDirectory.convention(build.dir("python-standalone")) } val xcframeworkSource = download.flatMap { task -> task.workDirectory.dir("extracted/ios/Python.xcframework") } @@ -245,6 +301,7 @@ internal fun PythonBindingsPlugin.configureIosConsumer(project: Project, extensi xcframework.set(xcframeworkSource) sliceName.set(slice.name) pythonVersion.set(DEFAULT_PYTHON_VERSION) + this.freeThreaded.set(freeThreaded) destinationDir.set(homeRoot.map { it.dir(slice.name) }) } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt index 5fef66032..43d946465 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt @@ -116,11 +116,14 @@ object IosPythonHomeLayout { ) } - /** `3.14` from `3.14.7`. */ - fun stdlibTag(pythonVersion: String): String { + /** + * `3.14` from `3.14.7`; `3.15t` from `3.15.0` for a free-threaded framework (SPEC T-2a), whose + * standard library is `lib/python3.15t/` and whose slices carry `lib-/python3.15t/`. + */ + fun stdlibTag(pythonVersion: String, freeThreaded: Boolean = false): String { val parts = pythonVersion.split('.') require(parts.size >= 2) { "python version '$pythonVersion' has no major.minor" } - return "${parts[0]}.${parts[1]}" + return "${parts[0]}.${parts[1]}" + if (freeThreaded) "t" else "" } /** @@ -128,8 +131,8 @@ object IosPythonHomeLayout { * first: the slice tree only adds files (`lib-dynload/`, `_sysconfigdata__ios_*`), but if it ever * overlapped, the slice-specific file is the one that must win, as in `install_stdlib`. */ - fun sourceDirectories(slice: Slice, pythonVersion: String): List { - val tag = stdlibTag(pythonVersion) + fun sourceDirectories(slice: Slice, pythonVersion: String, freeThreaded: Boolean = false): List { + val tag = stdlibTag(pythonVersion, freeThreaded) return listOf("lib/python$tag", "${slice.xcframeworkSlice}/lib-${slice.arch}/python$tag") } @@ -158,10 +161,10 @@ object IosPythonHomeLayout { * ([sourceDirectories]' order). Fails when a source tree is missing, rather than staging half a * prefix that would abort `Py_Initialize()` on the device. */ - fun plan(xcframework: File, slice: Slice, pythonVersion: String): Map { - val tag = stdlibTag(pythonVersion) + fun plan(xcframework: File, slice: Slice, pythonVersion: String, freeThreaded: Boolean = false): Map { + val tag = stdlibTag(pythonVersion, freeThreaded) val result = sortedMapOf() - for (relative in sourceDirectories(slice, pythonVersion)) { + for (relative in sourceDirectories(slice, pythonVersion, freeThreaded)) { val root = File(xcframework, relative) if (!root.isDirectory) { throw GradleException( @@ -204,6 +207,14 @@ abstract class StageIosPythonHomeTask : DefaultTask() { @get:Input abstract val pythonVersion: Property + /** + * Whether [xcframework] is a free-threaded build (`lib/pythont/`, SPEC T-2a). Unset means + * false, the GIL layout. + */ + @get:Input + @get:org.gradle.api.tasks.Optional + abstract val freeThreaded: Property + @get:OutputDirectory abstract val destinationDir: DirectoryProperty @@ -213,7 +224,8 @@ abstract class StageIosPythonHomeTask : DefaultTask() { ?: throw GradleException("unknown iOS slice '${sliceName.get()}'") val framework = xcframework.get().asFile val version = pythonVersion.get() - val plan = IosPythonHomeLayout.plan(framework, slice, version) + val ft = freeThreaded.getOrElse(false) + val plan = IosPythonHomeLayout.plan(framework, slice, version, ft) val root = destinationDir.get().asFile val home = File(root, PREFIX_DIRECTORY) @@ -233,7 +245,7 @@ abstract class StageIosPythonHomeTask : DefaultTask() { Files.copy(template.toPath(), File(root, TEMPLATE_NAME).toPath(), StandardCopyOption.REPLACE_EXISTING) // The marker PythonHomeCheck probes on the device, checked here where it is cheap to fail. - val marker = File(home, "lib/python${IosPythonHomeLayout.stdlibTag(version)}/os.py") + val marker = File(home, "lib/python${IosPythonHomeLayout.stdlibTag(version, ft)}/os.py") if (!marker.isFile) { throw GradleException("staged ${slice.name} into $home but $marker is missing; the app would abort at Py_Initialize().") } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt index 645ee524e..141c59bff 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt @@ -160,6 +160,63 @@ class IosConsumerWiringTest { assertEquals(File(build, "python-ios-home/${IosPythonHomeLayout.slices[0].name}"), stage.destinationDir.get().asFile) } + // ---- free-threaded archive (SPEC T-2a, #328 step 1) --------------------------------------- + + @Test + fun theDefaultIsStillTheGilArchiveAndLayout() { + // Guard, not a red test: step 1 must not switch the mobile default (#328 step 3 does). + val project = project() + wire(project) + val task = project.tasks.getByName(DOWNLOAD_IOS_TASK) as DownloadPythonIosTask + assertEquals(DEFAULT_IOS_ARCHIVE_URL, task.archiveUrl.get()) + assertEquals(DEFAULT_IOS_ARCHIVE_NAME, task.archiveName.get()) + assertEquals(DEFAULT_IOS_ARCHIVE_SHA256, task.expectedSha256.get()) + for (slice in IosPythonHomeLayout.slices) { + val stage = project.tasks.getByName(slice.stageTaskName) as StageIosPythonHomeTask + assertEquals(false, stage.freeThreaded.get(), slice.name) + } + } + + @Test + fun freeThreadedSelectsTheProjectsOwnReleaseArchive() { + val project = project() + wire(project) { ios { freeThreaded.set(true) } } + val v = DEFAULT_PYTHON_VERSION + val task = project.tasks.getByName(DOWNLOAD_IOS_TASK) as DownloadPythonIosTask + assertEquals( + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-${v}t-mobile/python-${v}t-iOS-XCframework.tar.gz", + task.archiveUrl.get(), + ) + assertEquals("python-${v}t-iOS-XCframework.tar.gz", task.archiveName.get()) + assertEquals(DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256, task.expectedSha256.get()) + for (slice in IosPythonHomeLayout.slices) { + val stage = project.tasks.getByName(slice.stageTaskName) as StageIosPythonHomeTask + assertEquals(true, stage.freeThreaded.get(), slice.name) + } + } + + @Test + fun theFreeThreadedPinIsTheReleasesDigest() { + // The value the release's SHA256SUMS and the GitHub asset digest both give for + // python-3.15.0t-iOS-XCframework.tar.gz (cpython-3.15.0t-mobile, run 39014667720, #163). + assertEquals("ios-$DEFAULT_PYTHON_VERSION-freethreaded", DEFAULT_IOS_FREE_THREADED_LOCK_KEY) + assertEquals(DEFAULT_IOS_FREE_THREADED_LOCK_KEY, FreeThreadedMobileRelease.lockKey("ios", DEFAULT_PYTHON_VERSION)) + if (DEFAULT_PYTHON_VERSION == "3.15.0") { + assertEquals("b8c0a4b32d31738f6ead7ace28acee0cb2a590bd0ca7015e9c32af613445e6b5", DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) + } + } + + @Test + fun theReleaseNamesFollowTheTSuffix() { + assertEquals("cpython-3.15.0t-mobile", FreeThreadedMobileRelease.tag("3.15.0")) + assertEquals("python-3.15.0t-iOS-XCframework.tar.gz", FreeThreadedMobileRelease.iosArchiveName("3.15.0")) + assertEquals( + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-3.15.0t-mobile/python-3.15.0t-aarch64-linux-android.tar.gz", + FreeThreadedMobileRelease.url("3.15.0", FreeThreadedMobileRelease.androidArchiveName("3.15.0", "aarch64")), + ) + assertEquals("android-aarch64-3.15.0-freethreaded", FreeThreadedMobileRelease.lockKey("android-aarch64", "3.15.0")) + } + // ---- download task ------------------------------------------------------------------------ private fun tarGz(entries: Map): ByteArray { @@ -221,6 +278,16 @@ class IosConsumerWiringTest { assertFalse(File(task.extractDirectory, "Python.xcframework").exists()) } + @Test + fun aMissingPinFailsBeforeAnyDownload() { + // A plugin built from a python-checksums.properties without the free-threaded key bakes "". + val bytes = tarGz(mapOf("Python.xcframework/Info.plist" to "plist")) + val (task, _) = downloadTask(bytes, "") + val e = assertFailsWith { task.provision() } + assertTrue("no pinned SHA-256" in e.message.orEmpty(), e.message) + assertFalse(File(task.workDirectory.get().asFile, "fake-ios.tar.gz").exists()) + } + // ---- script ------------------------------------------------------------------------------- @Test diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt index 7f11258e3..5c3d01df6 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt @@ -184,6 +184,97 @@ class IosPythonHomeLayoutTest { assertFalse(File(out, "home/${StageIosPythonHomeTask.TEMPLATE_NAME}").exists()) } + // ---- free-threaded (SPEC T-2a, #328) ---- + + @Test + fun freeThreadedSourceDirectoriesCarryTheTSuffix() { + val sim = IosPythonHomeLayout.sliceFor("-iphonesimulator", "x86_64") + assertEquals( + listOf("lib/python3.15t", "ios-arm64_x86_64-simulator/lib-x86_64/python3.15t"), + IosPythonHomeLayout.sourceDirectories(sim, "3.15.0", freeThreaded = true), + ) + assertEquals("3.15t", IosPythonHomeLayout.stdlibTag("3.15.0", freeThreaded = true)) + assertEquals("3.15", IosPythonHomeLayout.stdlibTag("3.15.0")) + } + + @Test + fun aFreeThreadedPlanStagesLibPython315t() { + val xcframework = fakeFreeThreadedXcframework() + val slice = IosPythonHomeLayout.sliceFor("-iphonesimulator", "arm64") + val plan = IosPythonHomeLayout.plan(xcframework, slice, "3.15.0", freeThreaded = true) + assertEquals( + listOf( + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", + "lib/python3.15t/os.py", + ), + plan.keys.toList(), + ) + assertFalse(plan.values.any { it.name.startsWith("libpython") }, "no libpython*.dylib: ${plan.values}") + } + + @Test + fun aGilPlanAgainstAFreeThreadedTreeFails() { + // Guard, not a red test: the GIL tag must not silently find a free-threaded tree. + val xcframework = fakeFreeThreadedXcframework() + assertFailsWith { + IosPythonHomeLayout.plan(xcframework, IosPythonHomeLayout.sliceFor("-iphoneos", "arm64"), "3.15.0") + } + } + + @Test + fun theTaskStagesAFreeThreadedPrefix() { + val xcframework = fakeFreeThreadedXcframework() + val project = ProjectBuilder.builder() + .withProjectDir(File(scratch, "project").apply { mkdirs() }) + .withGradleUserHomeDir(File(scratch, "gradle-home")) + .build() + val out = File(scratch, "out-ft") + val task = project.tasks.create("stageFt", StageIosPythonHomeTask::class.java) + task.xcframework.set(xcframework) + task.sliceName.set("iphoneos-arm64") + task.pythonVersion.set("3.15.0") + task.freeThreaded.set(true) + task.destinationDir.set(out) + task.stage() + + val staged = File(out, "home").walkTopDown().filter { it.isFile } + .map { it.relativeTo(File(out, "home")).invariantSeparatorsPath }.sorted().toList() + assertEquals( + listOf( + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphoneos.so", + "lib/python3.15t/os.py", + ), + staged, + ) + } + + /** + * The shape of `python-3.15.0t-iOS-XCframework.tar.gz` (release `cpython-3.15.0t-mobile`), taken + * from `tar tzf` of the published archive: `lib/python3.15t` shared, `lib-/python3.15t` per + * slice, `libpython3.15t.dylib` beside them. + */ + private fun fakeFreeThreadedXcframework(): File { + val root = File(scratch, "ft/Python.xcframework") + fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } + file("lib/python3.15t/os.py") + file("lib/python3.15t/encodings/__init__.py") + file("lib/python3.15t/test/test_os/test_os.py") + file("ios-arm64/lib/libpython3.15t.dylib") + file("ios-arm64/lib-arm64/libpython3.15t.dylib") + file("ios-arm64/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py") + file("ios-arm64/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphoneos.so") + file("ios-arm64_x86_64-simulator/lib/libpython3.15t.dylib") + file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py") + file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") + file("ios-arm64_x86_64-simulator/lib-x86_64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") + file("build/iOS-dylib-Info-template.plist") + return root + } + /** The observed 3.14 archive's shape, a few files deep. */ private fun fakeXcframework(): File { val root = File(scratch, "Python.xcframework") From e066013c88c94f92a7483d528e9a96b91bee7930 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:10:34 +0900 Subject: [PATCH 32/61] Docs: the last pip install example becomes uv pip install (#136) --- docs/investigations/typedpython-kernel-ir.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/investigations/typedpython-kernel-ir.md b/docs/investigations/typedpython-kernel-ir.md index 76d438456..5e566c90b 100644 --- a/docs/investigations/typedpython-kernel-ir.md +++ b/docs/investigations/typedpython-kernel-ir.md @@ -16,7 +16,7 @@ ### 1.1 MLIR Python 바인딩 wheel - [V] PyPI 의 `mlir-python-bindings` 는 **이름 선점용 빈 placeholder** (0.0.1, "not the real distribution"). `mlir` 도 2023 년 빈 패키지. 즉 `uv add mlir-python-bindings` 는 진짜를 주지 않는다 (`curl pypi.org/pypi//json`). -- [V] 실제 wheel 은 GitHub release `llvm/eudsl` 태그 `mlir-python-bindings` 에 있다 (GitHub API 로 목록 조회: 658 assets, 최신 `mlir_python_bindings-20261004+758aa4a92-*`, 거의 매일 갱신). 설치 가이드: `pip install mlir-python-bindings -f https://llvm.github.io/eudsl` (eudsl README 요약). 이전 저장소 `makslevental/mlir-wheels` 는 2025-11-29 archive, eudsl 로 이전됨 [V WebFetch]. +- [V] 실제 wheel 은 GitHub release `llvm/eudsl` 태그 `mlir-python-bindings` 에 있다 (GitHub API 로 목록 조회: 658 assets, 최신 `mlir_python_bindings-20261004+758aa4a92-*`, 거의 매일 갱신). 설치: `uv pip install mlir-python-bindings -f https://llvm.github.io/eudsl` (eudsl README 의 설치 안내를 uv 로 옮긴 것). 이전 저장소 `makslevental/mlir-wheels` 는 2025-11-29 archive, eudsl 로 이전됨 [V WebFetch]. - [V] 플랫폼/파이썬 매트릭스 (asset 이름 집계): cp310, 311, 312, 313, 314, **cp314t(free-threaded)** 각각 macOS arm64(`macosx_13_0_arm64`), Linux x86_64/aarch64(manylinux_2_34), Windows amd64. **cp315 / cp315t wheel 은 없음** (`cp315` 문자열 asset 0개). 크기 wheel 당 58 ~ 76 MB, 설치 후 `mlir/` 디렉터리 253 MB (macOS cp314t). - [V] 실제로 설치해 실행: `uv venv venv314t --python 3.14t; uv pip install --python venv314t/bin/python ` 성공. `sys.version` = "3.14.7 free-threading build", `import mlir` 후 `sys._is_gil_enabled()` = **False** (GIL 이 다시 켜지지 않음). 등록된 dialect: spirv, gpu, nvvm, vector, linalg, transform, amdgpu, x86, nvgpu, emitc, llvm, scf, memref, tensor, tosa 등 (`pkgutil.iter_modules(mlir.dialects.__path__)`). - [V] 공식 문서(mlir.llvm.org/docs/Bindings/Python/): free-threading 은 3.13+ 에서 "독립된 Context 라면 멀티스레드 안전". 안전하지 않다고 명시한 것: IR printing 과 PassManager 동시 사용, location error emit, module dump, transform interpreter, **GPU dialect 관련 작업**. 우리는 컴파일을 빌드 타임에 단일 프로세스/스레드로 돌리면 되므로 영향이 작다 [I]. From c30ff174f104e424077b83f89265a9952062b104 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:11:50 +0900 Subject: [PATCH 33/61] TypedPython: tests for three or more list parameters on free-threaded builds (#255, red) Tests first: 3- and 7-list functions compile and match the interpreter, concurrent writers lose no update, a caller-held critical section does not deadlock, an option (A) mutant loses updates. nbody's pinned compiled set becomes {advance, energy}. --- .../src/test/python/test_class_methods.py | 4 +- .../src/test/python/test_free_threaded.py | 296 ++++++++++++++++-- .../src/test/python/test_frontend.py | 3 +- 3 files changed, 276 insertions(+), 27 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index b35b5db6e..feb440897 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -920,7 +920,9 @@ def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): "fannkuch": {"fannkuch"}, "spectral_norm": {"eval_a", "mul_atv", "mul_av"}, "wordfreq": {"next_state", "word_of", "main"}, # word_of and main: native ints, #232 - "nbody": set(), + # #255: three or more lists hold every list's lock; offset_momentum (4 lists) reads the entry + # global SOLAR_MASS, which could wait while they are held (verify/array-locks): interpreted + "nbody": {"advance", "energy"}, } diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index a1e925f04..fe0d81b66 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -4,7 +4,8 @@ * declare `Py_mod_gil = Py_MOD_GIL_NOT_USED`, so importing it does not switch the GIL back on; * keep its shared mutable state thread-safe (runtime/API.md "Free-threaded CPython"); * read nothing through a borrowed reference that another thread can free; - * hold the list's critical section from before the copy-in to after the write-back (array params). + * hold the list's critical section from before the copy-in to after the write-back (array params); + with three or more list parameters, hold every list's lock for that whole window (#255). Every concurrency scenario runs in a CHILD interpreter with a timeout (a crash or a deadlock fails one test, not the run). Iteration counts scale with TP_FT_STRESS (default 1). @@ -12,6 +13,10 @@ Red record (before the change, on the 3.15t venv): the GIL test and the three module-level checks failed (`Py_mod_gil` absent, the GIL came back at import), the concurrent counters lost updates, and a three-list function was compiled. + +#255 (three or more lists, design/typedpython.md 4.3.6): red before the change, the functions with three +and seven list parameters (`three`, `bump3`, `seven`) were not compiled, so `built` failed and every test +of this file using it errored; the front-end, verifier and cgen tests of the 3-list section failed. """ from __future__ import annotations @@ -148,9 +153,50 @@ def three(a: list[int], b: list[int], c: list[int]) -> int: return len(a) + len(b) + len(c) +@compiled +def bump1(a: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + + +@compiled +def bump2(a: list[float], b: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + for i in range(len(b)): + b[i] = b[i] + 1.0 + + +@compiled +def bump3(a: list[float], b: list[float], c: list[float]) -> None: + for i in range(len(a)): + a[i] = a[i] + 1.0 + for i in range(len(b)): + b[i] = b[i] + 1.0 + for i in range(len(c)): + c[i] = c[i] + 1.0 + + +@compiled +def seven(x: list[float], y: list[float], z: list[float], vx: list[float], vy: list[float], + vz: list[float], w: list[float]) -> float: + e: float = 0.0 + for i in range(len(x)): + vx[i] += x[i] * w[i] + vy[i] -= y[i] * w[i] + vz[i] += z[i] * w[i] * 0.5 + e += vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i] + return e + + K: int = 3 +@compiled +def three_k(a: list[int], b: list[int], c: list[int]) -> int: + return len(a) + len(b) + len(c) + K + + @compiled def usek(x: int) -> int: return x + K @@ -196,7 +242,7 @@ def built(tmp_path_factory): assert result.extension is not None, result.skipped compiled = {f.name for f in result.module.functions} wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2", - "pair_edge", "usek", "fresh", "vpick"} + "pair_edge", "usek", "fresh", "vpick", "three", "bump1", "bump2", "bump3", "seven"} assert wanted <= compiled, {n: result.skipped.get(n) for n in wanted - compiled} fns = {f.name: f for f in result.module.functions} assert fns["vpick"].virtual == ("c",) and fns["fresh"].virtual == () # #143: one virtual, one real @@ -250,50 +296,124 @@ def test_no_runtime_warning_about_the_gil(extension): assert "global interpreter lock" not in proc.stderr -# --- more than two list parameters stay interpreted -------------------------------------------------- +# --- three or more list parameters (#255) ----------------------------------------------------------- +# Since #209 a function with more than two list parameters stayed interpreted (CPython's critical +# sections cover one or two objects). #255 holds every list's own lock for the whole call instead +# (design/typedpython.md 4.3.6). Red before #255: all of the tests below. -def test_three_list_parameters_stay_interpreted(built): +def test_three_and_seven_list_parameters_compile(built): result, _, _ = built - assert "three" not in {f.name for f in result.module.functions} - reason = result.skipped["three"] - assert "list" in reason and ("three" in reason or "more than two" in reason), reason + assert {"three", "bump3", "seven"} <= {f.name for f in result.module.functions}, result.skipped -def test_front_end_gives_a_clear_reason(tmp_path): +def test_front_end_lowers_three_list_parameters(tmp_path): p = tmp_path / "m.py" p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" " return len(a)\n") m = frontend.lower(p) - assert not m.functions - assert "more than two list parameters" in m.skipped["f"], m.skipped["f"] + assert [f.name for f in m.functions] == ["f"], m.skipped def test_two_list_parameters_still_compile(built): result, _, _ = built - assert {"zero2", "pair_edge"} <= {f.name for f in result.module.functions} + assert {"zero2", "pair_edge", "bump2"} <= {f.name for f in result.module.functions} -def test_verifier_rejects_three_array_params(built): - from typedpython import ir +def test_two_lists_keep_the_critical_section_and_three_take_the_locks(built): + """One or two lists: CPython's own PyCriticalSection / PyCriticalSection2, unchanged. Three or + more: tp_cs_begin_n, which takes every list's lock (and can refuse, then the call deopts).""" + result, path, _ = built + c_text = cgen.generate(result.module, path) + def wrapper(name): + start = c_text.index(f"static PyObject *tp_wrap_{name}(") + return c_text[start:c_text.index("\n}\n", start)] + assert "tp_cs_begin(&tp_lock" in wrapper("zero2") and "tp_cs_begin_n" not in wrapper("zero2") + assert "tp_cs_begin(&tp_lock" in wrapper("fill") and "tp_cs_begin_n" not in wrapper("fill") + for name, n in (("bump3", 3), ("seven", 7)): + w = wrapper(name) + assert f"if (tp_cs_begin_n(&tp_lock, tp_lobjs, {n}) != 0) goto tp_deopt;" in w, w + assert "_exit_held(&tp_a" in w and "_array_exit(&tp_a" not in w, w + + +def test_three_lists_with_an_entry_global_stay_interpreted(built): + """Nothing between the copy-in and the write-back may wait while three or more list locks are held + (an entry global read can take the module dict's critical section).""" result, _, _ = built + assert "three_k" not in {f.name for f in result.module.functions} + reason = result.skipped["three_k"] + assert "entry global" in reason and "three or more list" in reason, reason + + +def _with_third_list(result, *, entry_global: bool = False): + from typedpython import ir f = next(f for f in result.module.functions if f.name == "zero2") third = ir.ArrayParam("zs", ir.Type.I64_ARRAY, False) bad = dataclasses.replace(f, params=f.params + (third,)) - mod = dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) - proved, diags = verify.verify(mod) + if entry_global: + bad = dataclasses.replace(bad, entry_globals=(ir.Param("K", ir.Type.I64),)) + return dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) + + +def test_verifier_accepts_three_array_params(built): + result, _, _ = built + proved, diags = verify.verify(_with_third_list(result)) + assert [f.name for f in proved.functions] == ["zero2"], diags + + +def test_verifier_rejects_three_array_params_with_an_entry_global(built): + result, _, _ = built + proved, diags = verify.verify(_with_third_list(result, entry_global=True)) assert not proved.functions - assert any("array" in d.message and "two" in d.message for d in diags), diags + assert any(d.rule == verify.ARRAY_LOCKS and "entry global" in d.message for d in diags), diags -def test_cgen_refuses_three_array_params(built): - from typedpython import ir +def test_cgen_takes_three_locks(built): result, path, _ = built - f = next(f for f in result.module.functions if f.name == "zero2") - third = ir.ArrayParam("zs", ir.Type.I64_ARRAY, False) - bad = dataclasses.replace(f, params=f.params + (third,)) - mod = dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) - with pytest.raises(cgen.CGenError, match="two"): - cgen.generate(mod, path) + c_text = cgen.generate(_with_third_list(result), path) + assert "tp_cs_begin_n(&tp_lock, tp_lobjs, 3)" in c_text + + +def test_cgen_refuses_three_array_params_with_an_entry_global(built): + result, path, _ = built + with pytest.raises(cgen.CGenError, match="entry global"): + cgen.generate(_with_third_list(result, entry_global=True), path) + + +def test_three_and_seven_lists_match_the_interpreter(extension): + r = run_child(extension, """ + import random + rng = random.Random(255) + interp = m.__typedpython_interpreted__ + before = m.__typedpython_deopts__ + bad = [] + for n in (0, 1, 5, 300): + ls = [[rng.uniform(-3.0, 3.0) for _ in range(n)] for _ in range(7)] + a = [list(l) for l in ls] + b = [list(l) for l in ls] + ra, rb = m.seven(*a), interp["seven"](*b) + if ra != rb or a != b: + bad.append(("seven", n)) + a = [list(l) for l in ls[:3]] + b = [list(l) for l in ls[:3]] + m.bump3(*a) + interp["bump3"](*b) + if a != b: + bad.append(("bump3", n)) + ints = [list(range(n)), list(range(n + 1)), []] + if m.three(*ints) != interp["three"](*ints): + bad.append(("three", n)) + compiled_deopts = m.__typedpython_deopts__ - before + # deopts before any lock: a non-list argument and an aliased list run interpreted + x, y = [1.0, 2.0], [3.0] + m.bump3(x, x, y) + x2, y2 = [1.0, 2.0], [3.0] + interp["bump3"](x2, x2, y2) + same_alias = (x, y) == (x2, y2) + tup = m.three([1], (2, 3), [4]) + print(json.dumps({"bad": bad, "deopts": compiled_deopts, "alias": same_alias, "tuple": tup, + "deopted": m.__typedpython_deopts__ - before - compiled_deopts})) + """) + assert r == {"bad": [], "deopts": 0, "alias": True, "tuple": 4, "deopted": 2} # --- concurrent calls -------------------------------------------------------------------------------- @@ -435,6 +555,134 @@ def look(): assert r == {"bad": [], "errs": []} +# Three shared lists (#255). Every call adds 1.0 to every element of every list it is given, and each +# list is given to exactly three calls per round, so after R rounds every element is 3R exactly. A call +# whose lists are not all held from the copy-in to the write-back loses the increments another thread +# made in between (the write-back overwrites them): option (A) of #255, the mutant below. The lists stay +# below TP_POLL_INTERVAL, so no call polls (a poll writes back and copies in again, which keeps the +# count exact too, but would make the mutant's window smaller). +THREE_LISTS_BODY = """ + n = 3000 + a = [0.0] * n + b = [0.0] * n + c = [0.0] * n + R = ROUNDS + before = m.__typedpython_deopts__ + def t3(): + for _ in range(R): + m.bump3(a, b, c) + def t3r(): + for _ in range(R): + m.bump3(c, a, b) # another parameter order: the locks are taken in address order + def t1(): + for _ in range(R): + m.bump1(a) # a one-list critical section on a list held by bump3 + def t2(): + for _ in range(R): + m.bump2(c, b) # a two-list critical section (PyCriticalSection2) + errs = run_threads([t3, t3r, t1, t2]) + want = float(3 * R) + bad = [name for name, l in (("a", a), ("b", b), ("c", c)) if l != [want] * n] + lost = sum(int(want - v) for l in (a, b, c) for v in l) + print(json.dumps({"bad": bad, "lost": lost, "errs": errs, + "deopts": m.__typedpython_deopts__ - before})) +""" + + +def test_three_shared_lists_with_concurrent_writers(extension): + r = run_child(extension, THREE_LISTS_BODY.replace("ROUNDS", "(150 * STRESS)")) + assert r == {"bad": [], "lost": 0, "errs": [], "deopts": 0}, r + + +def test_seven_lists_and_a_concurrent_writer(extension): + """nbody's shape: one thread steps seven lists, another adds to one of them through a one-list + critical section; the result equals running the same calls one after the other in that order.""" + r = run_child(extension, """ + n = 2000 + ls = [[1.0] * n for _ in range(7)] + R = 100 * STRESS + before = m.__typedpython_deopts__ + def stepper(): + for _ in range(R): + m.bump3(ls[0], ls[3], ls[6]) + m.seven(*ls) + def writer(): + for _ in range(R): + m.bump1(ls[6]) + errs = run_threads([stepper, writer]) + print(json.dumps({"w": sorted(set(ls[6])), "errs": errs, "deopts": m.__typedpython_deopts__ - before})) + """) + assert r == {"w": [float(1 + 2 * 100 * STRESS)], "errs": [], "deopts": 0}, r + + +def _option_a_mutant(built, tmp: Path) -> Path: + """Option (A) of #255: bump3 holds its lists' locks at the copy-in and at the write-back only (the + wrapper drops them for the impl and takes them again before the write-back).""" + result, path, _ = built + c_text = cgen.generate(result.module, path) + pat = re.compile(r"^( tp_s = tp_impl_bump3\(.*\);)$", re.M) + assert len(pat.findall(c_text)) == 1, "bump3's impl call not found in the generated C" + mutated = pat.sub(lambda mo: " tp_cs_end(&tp_lock);\n" + mo.group(1) + + "\n (void)tp_cs_begin_n(&tp_lock, tp_lobjs, 3);", c_text) + return cbuild.build(mutated, "ftmod", tmp / "option_a", ext_suffix=None) + + +@pytest.mark.race_mutant +def test_mutant_option_a_locks_at_copy_in_and_write_back_only_is_caught(built, tmp_path): + so = _option_a_mutant(built, tmp_path) + lost = 0 + for _ in range(3): + r = run_child(so, THREE_LISTS_BODY.replace("ROUNDS", "300")) + lost = r["lost"] + if lost or r["bad"]: + break + assert lost > 0, "the option (A) mutant (no lock between copy-in and write-back) was not caught" + + +def test_caller_held_critical_section_does_not_deadlock(extension): + """list.sort holds the list's critical section while it calls the key (listobject.c, @critical_section + on list_sort_impl). A key that calls a three-list function with that list (the top-most section of + this thread) or with a list held by an outer, not top-most section must not wait for its own thread. + Results equal the interpreted function's in the same situation.""" + r = run_child(extension, """ + interp = m.__typedpython_interpreted__ + def run(fn): + out = {} + # (1) the sorted list itself, held by the top-most section of this thread + a = [3.0, 1.0, 2.0] + b = [0.0] * 4 + c = [0.0] * 4 + a.sort(key=lambda v: (fn(a, b, c), v)[1]) + out["top"] = (a, b, c) + # (2) a list held by an outer section while an inner sort's section is the top-most + outer = [2.0, 1.0] + inner = [5.0, 4.0, 6.0] + b2 = [0.0] * 3 + b3 = [0.0] * 2 + c3 = [0.0] * 5 + def k_inner(w): + fn(outer, b2, inner) # both sorted lists: one held by the top-most section + fn(outer, b3, c3) # only the outer one: held by a section below the top + return w + def k_outer(v): + inner.sort(key=k_inner) + return v + outer.sort(key=k_outer) + out["nested"] = (outer, inner, b2, b3, c3) + # (3) the list in the middle of the parameter list + d = [0.0] * 2 + e = [9.0, 8.0] + f = [0.0] * 2 + e.sort(key=lambda v: (fn(d, e, f), v)[1]) + out["middle"] = (d, e, f) + return out + got = run(m.bump3) + want = run(interp["bump3"]) + print(json.dumps({"same": got == want, "got": repr(got)})) + """, timeout=120) + assert r["same"], r["got"] + + def test_shared_objects_concurrent_fieldset_and_fieldget(extension): r = run_child(extension, """ cell = m.Cell((0,), None) diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index cd631efa6..9886beff0 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -329,8 +329,7 @@ def scale(a: list[float], b: list[float], k: float) -> None: a[i] = b[i] * k """) f = function(m, "scale") - # two list parameters at most (#159: the lists are locked from copy-in to write-back; CPython - # has no three-object critical section). A third one is test_free_threaded.py's. + # (#255: any number of list parameters; three or more are test_free_threaded.py's) assert f.params == ( ArrayParam("a", Type.F64_ARRAY, True), ArrayParam("b", Type.F64_ARRAY, False), Param("k", F64), ) From 89e9bfaeb8abf62353264f61710f395f9befcc05 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:12:19 +0900 Subject: [PATCH 34/61] TypedPython: lock three or more list parameters on free-threaded builds (#255) tp_cs_begin_n holds one list through a critical section and only tries the others' ob_mutex, restarting on the taken list and deopting after TP_CS_ROUNDS rounds; the write-back stores directly. verify/array-locks keeps entry globals and redo calls out of such functions. Agent proposal, not confirmed: relies on ob_mutex, which CPython documents as reserved (design 4.3.6). --- docs/SPEC.md | 2 + docs/design/typedpython.md | 132 +++++++++++++++ .../src/main/python/typedpython/cgen.py | 89 +++++++--- .../src/main/python/typedpython/frontend.py | 10 +- .../main/python/typedpython/runtime/API.md | 39 +++-- .../python/typedpython/runtime/tp_runtime.h | 158 ++++++++++++++++-- .../src/main/python/typedpython/verify.py | 44 ++++- 7 files changed, 418 insertions(+), 56 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 90d07e494..573302097 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -740,6 +740,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `continue` and `return` meaning exactly what they mean without chunking (trip counts just below, at and above a multiple of the interval, negative steps, empty ranges and ranges spanning the i64 limits all behave as CPython's `range`). A function holding an array copy-in or entry-globals snapshot polls too (#152, option (i)): at the top of a loop iteration, once per `TP_POLL_INTERVAL`, it writes its copied lists back (CPython's own stores would already be visible there), ends their critical sections, polls, takes the critical sections again, copies the lists in again and re-checks the entry guards (list lengths, exact element types, entry globals). Unchanged: it continues compiled, which is an execution CPython can produce, since every list operation of another thread now falls before or after this point. Changed: it leaves for the interpreter at that point (an OSR-style exit) and does not deopt or raise: a continuation function generated from the function's own source, holding the rest of the function from that loop iteration (the innermost loop from its current index; each enclosing loop's remaining body, then its remaining iterations; then the statements after), receives every live local (a native value as its Python object, a local array as a new list) and each enclosing `range`'s already evaluated bounds, and the compiled call returns what it returns or raises what it raises. The continuation keeps the original file, line numbers and function name, so a traceback reads as the interpreted function's. + A function with list parameters holds the lists' locks from its copy-in to its write-back, so every list operation of another thread that takes the list's lock (every store, resize, `sort`, ...) falls before or after the call (or a poll above): one or two lists through CPython's critical section (#159), three or more through every list's own lock (#255; agent proposal, not confirmed: design §4.3.6), taken without waiting while another of them is held. When they cannot all be taken (two of them held by one critical section of the calling thread itself, e.g. a `list.sort` key) the call runs interpreted, before any effect; a poll that cannot take them again leaves for the interpreter as when a list changed. Nothing in such a call may wait while the locks are held, so a function with three or more list parameters and an entry global or a redo Call in its call tree stays interpreted (`verify/array-locks`; nbody's `offset_momentum`, which reads `SOLAR_MASS`). **Floating-point exactness and `@compiled(fastmath=True)` (#153, #332, owner decision 2026-10-25):** by default (`fastmath=False`), floating-point compilation is bit-identical to CPython (`#pragma STDC FP_CONTRACT OFF` in the generated C, `-ffp-contract=off -fno-fast-math` on GCC/Clang, `/fp:precise` @@ -776,6 +777,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. | `array.array[float]` / `array.array[int]` parameters (exact `array.array`, typecode `d` or `q`/`l`, 8-byte items) are read and written in place: the buffer export is held for the call and released on every exit, and every element access is one relaxed atomic 8-byte load or store, so each access is atomic like the interpreter's `a[i]`; no copy-in and no limit on the number of buffers. While the export is held, a resize of that array (`append`, `extend`, `del a[i]`, ... from another thread) raises BufferError, exactly as CPython does for any held export; the compiled call never sees a resize. Concurrent writes from another thread to the same elements are not synchronised: as in CPython, no ordering between the two threads' element accesses is guaranteed, only that each access is whole (no torn element). `memoryview[float]` / `memoryview[int]` parameters (#149: exact `memoryview`, 1-D, C-contiguous, format `d` or `q`/`l`, 8-byte aligned, writable when stored to; any exporter: an `array.array`, a numpy array, a `bytearray` cast with `.cast('d')`) take the same path with array.array's element semantics (an element is a Python float / int); the export is taken on the memoryview, so while the call runs `release()` on it raises BufferError, as CPython does for an exported view. An out-of-range index raises memoryview's `IndexError("index out of bounds on dimension 1")`. A released, read-only (when stored to), non-contiguous, multi-dimensional or other-format view deopts: the interpreter raises or runs it. numpy arrays: see the next row | anything else: object path | | numpy `npt.NDArray[np.float64]` / `NDArray[np.int64]` parameters (exact `numpy.ndarray`, 1-D, C-contiguous, native byte order, writable when stored to) take the same buffer path (#220, step 1) when the function accesses the array purely by element (`x[i]`, `len(x)`); a function that uses it as a whole object (`np.sum(x)`, `x.T`, passing `x` on, a slice) compiles as before with the parameter an opaque object. `NDArray[np.bool_]` (#261) is the same path with 1-byte items (buffer format `?`; a `uint8` or other array deopts): an element read is a native bool, boxed as `np.bool` when it escapes; a store compiles only for a bool value (`x[i] = v`, `v: bool`), because numpy converts an int or float by truthiness and that function stays interpreted. Other dtypes are opaque objects. Results follow the interpreter: a value derived from an element is a numpy scalar there, so it is boxed as `np.float64` / `np.int64` / `np.bool` when it escapes (return, `Box`), decided per value at run time by a taint flag; a function declared `-> float` / `int` / `bool` that holds a numpy array returns the interpreter's object. `float(v)` of such a value is a plain float. An out-of-range index raises numpy's `IndexError("index i is out of bounds for axis 0 with size n")`. Arithmetic on a value derived from an element follows numpy's scalar semantics (#220, step 2): float `/`, `//`, `%` never raise ZeroDivisionError (inf / nan, with numpy's result bits and sign rules, `//` and `%` as numpy's `divmod`), int64 `//` and `%` by zero give 0, int64 `/` converts both operands to double. numpy's RuntimeWarning (category and text exactly as numpy's, e.g. `overflow encountered in scalar add`, `divide by zero encountered in scalar divide`, `invalid value encountered in scalar floor_divide`) is emitted once per flagged operation, with the source file and line of the operation. A pure function does not emit it itself: on any floating-point exception (overflow, invalid, divide by zero) or int64 wrap or division by zero it deopts, so the interpreter emits it; an impure function cannot deopt after an effect, so the compiled code emits the warning (`PyErr_WarnExplicitObject`, honouring `warnings` filters and the module's `__warningregistry__`). Float `+ - * / // %` on such values is compiled in both kinds of function, including a store of the result into the array (`x[i] -= d * m`); int64 `/ // %` is compiled in a pure function only. Int64 `+ - *` and `// %` of an element in an impure function (a native INT would have to be a numpy scalar) stay on the object path | anything else: object path | | A function holding an array parameter runs no user code: closed, and no slot read or `C(args)` whose slow path runs user code, in its whole call tree (#203) | not compiled; diagnostic | + | A function with three or more list parameters waits for nothing while it holds their locks: no entry global and no redo Call in its whole call tree (#255) | not compiled; diagnostic (`verify/array-locks`) | | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | | Integer overflow checked and promoted (deopt); an op marked `proven` must re-prove its interval fits i64 | not compiled; diagnostic `verify/proven-overflow` | | `Call.redo` only from an impure caller to a pure, may-deopt callee returning F64/BOOL/NONE | not compiled; diagnostic `verify/redo` | diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index aee22b5d6..4d375c14c 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -467,6 +467,138 @@ i64/f64 지역)는 **컴파일된 구간 안의 임시 값**만 줄인다. 컨 - 효과가 작아 하지 않는 것: `-OO`(0.35~0.8 MB, `assert` 도 사라짐), deopt 원본 지연 로딩(함수 1000 개당 약 0.4 MB), 동결 모듈 끄기(오히려 4.4 MB 증가). +### 4.3.6 리스트 매개변수 세 개 이상: 리스트마다 잠금 (#255, 조사와 제안) + +**출처.** 이슈 #255 본문: "User decision (2026-10-17): research option (C) for the nbody regression from #209." +같은 본문: "Until (C) is settled, (B) stays." 오너가 정한 것은 (C) 를 **조사한다**는 것이고, 아래의 방식 +(C') 과 그 조건은 **에이전트 제안, 확인되지 않음**이다. + +**문제.** #209 이후 리스트 매개변수가 셋 이상인 함수는 3.15t 에서 해석 실행된다. copy-in 부터 write-back 까지 +모든 리스트의 잠금을 쥐어야 하는데, 공개 critical section API 는 객체 하나(`PyCriticalSection`) 또는 둘 +(`PyCriticalSection2`)만 잠근다. nbody 의 `offset_momentum`(리스트 4개), `advance`·`energy`(7개)가 31배에서 +1배로 떨어졌다. (A) copy-in 과 write-back 때만 잠그기는 그 사이 다른 스레드의 원소 쓰기를 잃고, (B) 지금처럼 +거절하기가 현재 상태다. + +#### 가. CPython 3.15.0rc1 의 근거 + +venv 의 3.15.0rc1t 헤더와, python/cpython 의 태그 `v3.15.0rc1` 소스(설치된 `pycore_lock.h` 와 바이트 단위로 +같음을 확인)에서 읽었다. 줄 번호는 그 태그 기준이다. + +1. **Begin.** 빠른 경로는 mutex 를 CAS 로 한 번 잡고 스레드의 섹션 스택에 올린다 + (`Include/internal/pycore_critical_section.h:106-117`). 실패하면 `_PyCriticalSection_BeginSlow` + (`Python/critical_section.c:20-62`): 같은 mutex 를 **맨 위** 섹션이 이미 쥐었으면 다시 잠그지 않고 돌아간다 + (29-45), world 가 멈춰 있으면 잠그지 않는다(46-54), 아니면 **새 섹션을 먼저 스택에 올리고** `PyMutex_Lock` + 으로 기다린다(55-60). 두 객체는 주소 순으로 잡고(`pycore_critical_section.h:163-170`), 느린 경로는 첫 mutex 를 + 쥔 채 둘째를 기다린다(`critical_section.c:91-101`). +2. **Detach 와 중단.** `PyMutex_Lock` 은 기다릴 때 스레드 상태를 놓는다(`Python/lock.c:654-656`, 138-139; + `Python/parking_lot.c:328-337`). 놓을 때 `detach_thread` 가 `_PyCriticalSection_SuspendAll` 을 불러 + (`Python/pystate.c:2285-2296`) **활성 섹션 전부**의 mutex 를 푼다(`critical_section.c:106-130`). 다시 붙을 때는 + **맨 위 섹션 하나만** 다시 잡는다(`pystate.c:2269-2273`, `critical_section.c:132-165`). 그래서 세 번째 섹션을 + 중첩하면 경합 때 바깥 둘이 풀린 채로 남는다(`Include/critical_section.h:16-22`: "This API does not provide a + way to lock more than two objects at once"). +3. **Stop-the-world.** `stop_the_world` 는 놓인(detached) 스레드를 SUSPENDED 로 바꾸고 붙어 있는 스레드에는 + eval breaker 비트만 세운다(`pystate.c:2385-2408`, 2409-2470). 기다리는 스레드는 놓인 상태이므로 GC 를 막지 + 않는다. world 가 멈춘 동안의 Begin 은 잠그지 않는다(`critical_section.c:46-54`, 69-74). GC 는 할당에서 예약만 + 하고(`Python/gc_free_threading.c:2031-2035`, 2696-2702) eval breaker 에서 돈다. +4. **`ob_mutex` 의 지위.** `PyObject` 의 필드다(`Include/object.h:162`, `Py_GIL_DISABLED` 일 때만; + `_Py_OPAQUE_PYOBJECT`, 곧 abi3t 에서는 숨겨진다). 문서는 "reserved for use by the critical section API ... Do + not lock it directly with `PyMutex_Lock`; doing so can cause deadlocks" 라고 한다 + (`Doc/c-api/structures.rst:58-66`, `Doc/howto/free-threading-extensions.rst:391-410`). `PyMutex` 의 비트(0b01 + 잠김, 0b10 대기자 있음)는 공개 헤더에 "not part of the public API, but described" 로 적혀 있다 + (`Include/cpython/pylock.h:26-35`). 한편 인터프리터 자신이 `ob_mutex` 를 섹션 밖에서 **시도만** 하고 실패하면 + 일반 경로로 간다: `LOCK_OBJECT` = `PyMutex_LockFast(&op->ob_mutex)` (`Python/ceval_macros.h:318-319`), + `STORE_SUBSCR_LIST_INT` (`Python/bytecodes.c:1431`), `list.append` 특수화(5114). 3.15.0rc1 에서 그 밖의 직접 + 사용은 faulthandler 덤프의 시간 제한 0 시도(`Python/pylifecycle.c:3517-3540`)와 섹션 API 를 거치는 + `typeobject.c:83` 뿐이다(`grep ob_mutex`). +5. **섹션을 여는 CPython 연산.** `PyList_SetItem` 은 리스트의 섹션을 연다(`Objects/listobject.c:454-479`). + dict 읽기는 실패 경로에서 dict 의 섹션을 연다(`Objects/dictobject.c:1647-1660`). `list.extend(dict)` 는 + `PyCriticalSection2(list, dict)` 다(`listobject.c:1489`). `list.sort` 는 key 를 부르는 동안 리스트의 섹션을 + 쥔다(`listobject.c:2925` `@critical_section`, 2993 의 key 호출). +6. **잠그지 않는 읽기.** 3.15 의 리스트 원소 읽기는 잠그지 않는다(`listobject.c:354-379`, `bytecodes.c:1148-1151`). + +#### 나. 판정 + +- **`PyMutex_Lock(&list->ob_mutex)` 를 주소 순으로 (원안 C): 건전하지 않다.** 두 가지가 깨진다. + (1) 자기 교착: 호출자가 그 리스트의 섹션을 쥔 채 우리를 부르면(`lst.sort(key=lambda v: f(lst, b, c))`) + `PyMutex_Lock` 이 기다리며 놓을 때 SuspendAll 이 그 섹션을 풀고, 다시 붙을 때 Resume 이 같은 mutex 를 잡으려 한다. + 공정성 handoff 로 mutex 가 이미 우리 몫이 되었으면 Resume 의 `PyMutex_Lock` 은 영원히 기다린다(그 안에서 다시 + 놓아도 맨 위 섹션이 비활성이라 풀 것이 없다, `critical_section.c:113`). handoff 가 없으면 깨어날 때마다 Resume 이 그 mutex 를 다시 잡아 우리 시도가 또 실패하는 일을 되풀이한다. 문서의 경고가 이 경우다. + (2) 잠근 채 기다림: 앞 리스트를 직접 쥔 채 다음 리스트를 기다리면 SuspendAll 은 직접 쥔 것을 풀지 않고, 다시 붙을 + 때 Resume 이 호출자 섹션의 mutex 를 주소 순서와 무관하게 잡는다. 주소 순 체인이 깨져 순환이 생길 수 있다. +- **공개 API 만으로: 같은 보장을 얻는 방법을 찾지 못했다.** 섹션을 중첩한 뒤 하나도 중단되지 않았는지 확인하려면 + 섹션 태그(`_cs_prev`, 비공개 필드)를 읽어야 한다. 내용 비교(double collect)는 리스트에 버전 번호가 없어 ABA 를 + 배제하지 못한다. `PyCriticalSection*_BeginMutex`(3.14 부터 공개, `Include/cpython/critical_section.h:38-42`)도 + mutex 둘까지다. 이 판단이 놓칠 수 있는 것: 3.15 이후에 다객체 섹션 API 가 생기면 이 절은 그 API 로 바꾼다. +- **제안 (C'), 에이전트 제안, 확인되지 않음.** `tp_cs_begin_n` (tp_runtime.h). 리스트를 주소 순으로 정렬하고 + 라운드마다 + 1. 리스트 하나(`w`)에 **CPython 섹션**을 연다. 기다려야 하면 CPython 규칙대로 기다린다(위 1, 2: 호출자 섹션은 + 기다리는 동안 풀리고, 맨 위 섹션이 쥔 리스트는 다시 잠그지 않는다); + 2. 나머지 리스트의 `ob_mutex` 는 **시도만** 한다(`_PyMutex_LockTimed` 의 첫 CAS, `lock.c:58-62`, 대기자 비트 유지); + 3. 하나라도 잡혀 있으면 직접 잡은 것을 풀고 섹션을 닫고, 다음 라운드는 그 리스트로 1 을 한다. 64 라운드 + (`TP_CS_ROUNDS`) 안에 못 가지면 아무것도 쥐지 않은 채 deopt 한다(효과 전, 해석 실행). + write-back 은 `PyList_SetItem` 대신 그 본문(색인 확인, release store, 옛 항목 해제)을 섹션 없이 한다 + (`tp_*_array_exit_held`): `PyList_SetItem` 은 직접 쥔 리스트에 섹션을 열어 자기 스레드를 기다린다(위 5). + 리스트 하나, 둘은 바뀌지 않는다(`PyCriticalSection`, `PyCriticalSection2`). +- **조건 (verifier `verify/array-locks`, cgen 도 다시 거절).** 리스트가 셋 이상인 함수는 호출 트리 전체에 + entry global 도 redo Call 도 없어야 한다. entry global 읽기는 dict 섹션을 기다릴 수 있고(위 5), redo Call 은 + 해석 실행이다. 잠금을 쥔 창 안에서 그런 대기가 생기면, 예를 들어 다른 스레드가 `lst.extend(globals())` 로 dict + 를 먼저 쥐고 `lst` 를 기다리는 동안 우리는 `lst` 를 쥐고 dict 를 기다린다. closed (#203) 는 이미 요구된다. + +#### 다. 증명한 것과 가정한 것 + +증명(위 소스에 근거한 논증): + +- **배제.** 다른 스레드의 섹션과 `LOCK_OBJECT` 는 같은 mutex 비트를 보므로, 우리가 쥔 리스트에서 기다리거나 + (`critical_section.c:59`) 일반 경로로 간다(`bytecodes.c:1431`). 잠그는 리스트 연산은 모두 호출 앞이나 뒤에 놓인다. +- **교착 없음.** 이 스레드는 mutex 를 직접 쥔 채로는 기다리지 않는다: 직접 잡기는 시도뿐이고, 기다림은 직접 쥔 것을 + 모두 푼 뒤 섹션 하나로만 한다. 창 안에는 기다리는 연산이 없다(조건 + closed + write-back 직접 저장). 그러니 창 + 동안 이 스레드에서 나가는 대기 간선이 없어 순환에 들 수 없다. 끝낼 때는 직접 쥔 것을 먼저 풀고 섹션을 닫는다 + (닫기가 바깥 섹션을 다시 잡으며 기다릴 수 있기 때문, `pycore_critical_section.h:128-137`). +- **섹션이 창 동안 중단되지 않음.** 중단은 detach(`pystate.c:2290-2292`) 와 기다리는 Begin 에서만 일어나고, 창 + 안에는 둘 다 없다. +- **재진입.** 호출자의 맨 위 섹션이 쥔 리스트는 그 섹션이 계속 쥔다(Begin 이 다시 잠그지 않음, 위 1). 아래 섹션이 + 쥔 리스트는 우리가 기다리는 동안 풀려 우리 것이 된다(CPython 의 중첩 의미 그대로). 맨 위의 + `PyCriticalSection2` 하나가 두 리스트를 쥔 경우만 영원히 못 가지며, 64 라운드 뒤 deopt 로 끝난다. +- **#152 의 성질 유지.** 긴 루프의 poll 은 write-back 하고 모든 잠금을 푼 뒤 poll 하고 다시 잡는다. 다시 못 잡으면 + 리스트가 바뀐 때처럼 해석 실행으로 넘어간다(OSR 출구). 창 안에서 detach 하지 않으므로 GC stop-the-world 는 그 + 호출(또는 다음 poll)을 기다린다: 1~2 리스트와 같은 liveness 지연이다. +- **GIL 빌드.** 모두 no-op 이다(`tp_cs_begin_n` 은 0 을 돌려준다). 두 헤더 집합(3.15t, 3.15)에 대해 + `clang -fsyntax-only` 로 생성 C 를 확인했다. + +가정(증명 아님): + +- **A1.** `ob_mutex` 의 위치와 `PyMutex` 비트 의미가 3.15 동안 바뀌지 않는다. 문서는 직접 사용을 금한다(위 4). + 우리는 문서가 경고하는 `PyMutex_Lock` 은 쓰지 않고, 인터프리터의 `LOCK_OBJECT` 와 같은 시도만 쓴다. 그래도 + "reserved" 를 넘는 사용이므로 **오너 결정이 필요하다.** +- **A2.** 창 안의 C 코드(복사, malloc, float/int 생성과 해제, 예외 객체 생성)는 기다리지 않는다. 소스 감사에 + 근거한 가정이고, 3.15 판이 바뀌면 다시 본다. +- **A3.** CPython 의 다른 코드는 `ob_mutex` 를 섹션 API 밖에서 기다리며 잡지 않는다(위 4 의 grep 결과). + +이 변경 범위 밖에서 찾은 것(1~2 리스트 경로에도 #255 이전부터 있는 성질, 별도 이슈 제안): + +- 1~2 리스트 함수가 창 안에서 entry global 을 읽다가 dict 섹션을 기다리면(위 5) 리스트의 섹션이 중단되어, 그 사이 + 다른 스레드의 쓰기를 write-back 이 덮어쓸 수 있다. redo Call 도 같다. +- 리스트 원소 읽기는 잠그지 않으므로(위 6) write-back 도중의 다른 스레드 읽기는 일부만 쓰인 상태를 볼 수 있다. + "호출 하나가 원자 단계" 는 잠그는 연산에 대해서만 성립한다. + +#### 라. 시험과 결과 + +- `test_free_threaded.py`: 리스트 3개·7개 함수가 컴파일되고 인터프리터와 같은 결과(빈 리스트, 별칭, 리스트가 아닌 + 인자 포함); 리스트 3개를 네 스레드가 서로 다른 순서와 1·2 리스트 함수로 동시에 더해도 증가를 하나도 잃지 않음; + `list.sort` 의 key 에서 부르는 경우(맨 위 섹션, 아래 섹션, 가운데 매개변수) 교착 없이 인터프리터와 같음; + entry global 이 있는 3 리스트 함수는 해석 실행. 변형 시험(`race_mutant`, CI 의 기본 실행 밖): 옵션 (A) + (copy-in 과 write-back 때만 잠금)는 같은 동시 시험에서 증가를 잃어야 한다. +- nbody: `advance`, `energy` 가 컴파일되고 `offset_momentum` 은 `SOLAR_MASS` (entry global) 때문에 해석 실행이다 + (`test_class_methods.py` 의 EXPECTED). 측정은 CI 의 결과를 기다린다. + +#### 마. 오너 결정이 필요한 점 + +1. (C') 를 받아들일지: `ob_mutex` 를 문서의 "reserved" 를 넘어 시도로 잡는다(A1). 받아들이지 않으면 (B) 로 + 돌린다(frontend/verify/cgen 의 조건 하나). +2. entry global 이 있는 3 리스트 이상 함수(nbody `offset_momentum`): 잠그기 전에 읽고 copy-in 뒤 전역 epoch 로 + 재검증하는 방안이 있으나 메모리 순서 논증이 끝나지 않았다. 지금은 해석 실행. +3. `TP_CS_ROUNDS` = 64 (에이전트 제안). + ### 4.4 Kotlin 경계의 직접 호출 이 플랫폼에서만 가능한 이점이다. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index fe1635e79..ae30cb1aa 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -25,11 +25,15 @@ Free-threaded CPython 3.15t (issue #159; runtime/API.md "Free-threaded CPython" holds the table of every piece of shared state). The module slots declare `{Py_mod_gil, Py_MOD_GIL_NOT_USED}` (under `#ifdef Py_GIL_DISABLED`), so importing it keeps the GIL off. In `tp_wrap_f`, a function with list -parameters holds the lists' critical section (`tp_cs_begin`, one list or two) from before the first -copy-in to after the last write-back: `tp_cs_end` is part of the same `exits` that run the write-back -on every path (ok, error, deopt, the last before the interpreted redo), and is a no-op on a path -that left before locking. More than two list parameters are refused here (`CGenError`), as in the -front end and the verifier: CPython has no three-object critical section. A function that holds an +parameters holds the lists' locks from before the first copy-in to after the last write-back: +`tp_cs_end` is part of the same `exits` that run the write-back on every path (ok, error, deopt, the +last before the interpreted redo), and is a no-op on a path that left before locking. One list or +two: CPython's critical section (`tp_cs_begin`). Three or more (#255, design/typedpython.md 4.3.6): +`tp_cs_begin_n` holds every list's own mutex; when it cannot take them (a list locked by this very +thread's caller) the call deopts before any effect, and the write-back is `tp_*_array_exit_held`. +Nothing in that window may wait, so a function with three or more lists and an entry global or a +redo Call in its call tree is refused here (`CGenError`), as in the verifier (verify/array-locks). +A function that holds an array does not poll the eval breaker (#141, open decision #152), so a long compiled array loop delays other threads' stop-the-world requests until it returns: a liveness delay, results unaffected. The `__typedpython_deopts__` read-add-write of `tp_cg_redo` runs in the module dict's @@ -922,8 +926,10 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: _check_entry_globals(f, functions) n_arrays = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params) if n_arrays > 2: - raise CGenError(f"{f.name}: {n_arrays} array parameters; at most two (CPython offers one- and " - "two-object critical sections only, and a third lock could suspend the others)") + why = _lock_window_waits(f, functions, set()) + if why is not None: + raise CGenError(f"{f.name}: {n_arrays} list parameters hold every list's lock from copy-in " + f"to write-back, so nothing in the call may wait, but {why} (#255)") for p in f.params: if isinstance(p, ir.ArrayParam): if p.type not in ir.ARRAYS: @@ -959,6 +965,26 @@ def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> return True +def _lock_window_waits(f: ir.Function, functions: dict[str, ir.Function], seen: set[str]) -> str | None: + """Why [f] or a function it calls can wait (an entry global read can take the module dict's + critical section; a redo Call runs the interpreted callee), or None (#255).""" + if f.name in seen: + return None + seen.add(f.name) + if f.entry_globals: + return f"'{f.name}' reads entry global '{f.entry_globals[0].name}'" + for e in _all_exprs(f): + if isinstance(e, ir.Call): + if e.redo: + return f"'{f.name}' has a redo Call of '{e.function}'" + callee = functions.get(e.function) + if callee is not None: + why = _lock_window_waits(callee, functions, seen) + if why is not None: + return why + return None + + def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> None: names = [g.name for g in f.entry_globals] if len(set(names)) != len(names): @@ -2338,13 +2364,15 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None [site] (the impl returns 2 with tp_my->result set). Only the call its wrapper entered polls.""" r = self.osr_sites_by_site[site] lists = self._list_params() + held = len(lists) > 2 # #255: every list's mutex, not a critical section + ex = "_exit_held" if held else "_exit" self.emit("if (tp_my != NULL) { /* #152: poll inside a snapshot holder */") self.depth += 1 self.emit("int tp_och = 0;") for j, p in enumerate(lists): ptr = _ident("l", p.name) self.emit(f"Py_ssize_t tp_ol{j} = {ptr}->len;") - self.emit(f"if ({_ARRAY_PFX[p.type]}_exit({ptr}) < 0) {{ tp_rc = -1; goto tp_exit; }}") + self.emit(f"if ({_ARRAY_PFX[p.type]}{ex}({ptr}) < 0) {{ tp_rc = -1; goto tp_exit; }}") if lists: self.emit("tp_cs_end(tp_my->lock);") self.emit("tp_t->snapshot_depth--;") @@ -2359,13 +2387,22 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None self.emit("#endif") self.emit("tp_t->snapshot_depth++;") self.emit("if (tp_s != 0) { tp_rc = -1; goto tp_exit; }") - if lists: - self.emit("tp_cs_begin(tp_my->lock, tp_my->lists[0], tp_my->lists[1]);") + if held: + # not taken again (a list locked by this thread's caller): leave for the continuation with + # nothing held (tp_och 2: no fresh copies). One statement per line: the race mutant of + # test_free_threaded.py drops every line that names tp_cs_begin. + self.emit("tp_s = 0;") + self.emit(f"tp_s = tp_cs_begin_n(tp_my->lock, tp_my->lock->objs, {len(lists)});") + self.emit("if (tp_s != 0) tp_och = 2;") + elif lists: + second = "tp_my->lists[1]" if len(lists) == 2 else "NULL" + self.emit(f"tp_cs_begin(tp_my->lock, tp_my->lists[0], {second});") for j, p in enumerate(lists): ptr = _ident("l", p.name) - self.emit(f"tp_s = {_ARRAY_PFX[p.type]}_enter(tp_my->lists[{j}], {ptr});") - self.emit("if (tp_s < 0) { tp_cs_end(tp_my->lock); tp_rc = -1; goto tp_exit; }") - self.emit(f"if (tp_s != 0 || {ptr}->len != tp_ol{j}) tp_och = 1;") + guard = "if (tp_och != 2) " if held else "" + self.emit(f"{guard}{{ tp_s = {_ARRAY_PFX[p.type]}_enter(tp_my->lists[{j}], {ptr});") + self.emit(" if (tp_s < 0) { tp_cs_end(tp_my->lock); tp_rc = -1; goto tp_exit; }") + self.emit(f" if (tp_s != 0 || {ptr}->len != tp_ol{j}) tp_och = 1; }}") for g in self.f.entry_globals: k = self.mg.name_index(g.name) lv = _ident("l", g.name) @@ -2380,8 +2417,8 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None self.emit("if (tp_och) { /* leave for the interpreter at this iteration (OSR-style exit) */") self.depth += 1 for j, p in enumerate(lists): - # fresh copies hold no dirty element: this exit only frees them - self.emit(f"(void){_ARRAY_PFX[p.type]}_exit({_ident('l', p.name)});") + # fresh copies hold no dirty element: this exit only frees them (a zeroed struct: no-op) + self.emit(f"(void){_ARRAY_PFX[p.type]}{ex}({_ident('l', p.name)});") if lists: self.emit("tp_cs_end(tp_my->lock);") n = len(r.params) @@ -2968,7 +3005,9 @@ def wrapper(self) -> str: args.append("&tp_r") # PyBuffer_Release first (cannot fail), then the lists' write-back and the critical section exits = [f" tp_buf_exit(&tp_a{i});" for i, p in buffers] - exits += [f" if ({_ARRAY_PFX[p.type]}_exit(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] + held = len(arrays) > 2 # #255: every list's mutex is held directly (tp_cs_begin_n) + ex = "_exit_held" if held else "_exit" + exits += [f" if ({_ARRAY_PFX[p.type]}{ex}(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] if arrays: exits.append(" tp_cs_end(&tp_lock); /* after the write-back; a no-op if never taken */") has_exits = bool(arrays or buffers) @@ -3028,8 +3067,14 @@ def wrapper(self) -> str: # as tp_*_array_enter would, without being locked. L.append(" if (" + " || ".join(f"!PyList_CheckExact(tp_args[{i}])" for i, _ in arrays) + ") goto tp_deopt;") - second = f"tp_args[{arrays[1][0]}]" if len(arrays) == 2 else "NULL" - L.append(f" tp_cs_begin(&tp_lock, tp_args[{arrays[0][0]}], {second});") + if held: + # the lists in address order (sorted in place by tp_cs_begin_n), kept for the polls + objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) + L.append(f" PyObject *tp_lobjs[{len(arrays)}] = {{{objs}}};") + L.append(f" if (tp_cs_begin_n(&tp_lock, tp_lobjs, {len(arrays)}) != 0) goto tp_deopt;") + else: + second = f"tp_args[{arrays[1][0]}]" if len(arrays) == 2 else "NULL" + L.append(f" tp_cs_begin(&tp_lock, tp_args[{arrays[0][0]}], {second});") for i, p in arrays: L.append(f" tp_s = {_ARRAY_PFX[p.type]}_enter(tp_args[{i}], &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") @@ -3057,10 +3102,12 @@ def wrapper(self) -> str: osr = self.osr if osr: lst = [i for i, p in arrays] - l0 = f"tp_args[{lst[0]}]" if lst else "NULL" - l1 = f"tp_args[{lst[1]}]" if len(lst) > 1 else "NULL" lock = "&tp_lock" if lst else "NULL" - L.append(f" tp_osr tp_ox = {{{lock}, {{{l0}, {l1}}}, NULL}};") + if lst: + # the list arguments in parameter order (tp_lobjs, the lock's own array, is sorted) + L.append(f" PyObject *const tp_olists[{len(lst)}] = {{" + + ", ".join(f"tp_args[{i}]" for i in lst) + "};") + L.append(f" tp_osr tp_ox = {{{lock}, {'tp_olists' if lst else 'NULL'}, NULL}};") L.append(" tp_t->osr = &tp_ox; /* #152: taken by the impl at entry */") L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module, tp_ts, tp_t{''.join(', ' + a for a in args)});") if osr: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 83fa2083d..8f81d2ca1 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -743,13 +743,9 @@ def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = No params.append(ir.Param(a.arg, Type.OBJ, c[0], c[1])) else: params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) - n_lists = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in params) - if n_lists > 2: - # Free-threaded CPython (#159): the lists are locked from copy-in to write-back, and CPython - # offers one- and two-object critical sections only (a nested third could suspend the others). - raise Skip(line, f"more than two list parameters ({n_lists}) are not supported: compiled code " - "holds each list's lock from copy-in to write-back, and free-threaded CPython " - "has no lock for more than two objects at once") + # Any number of list parameters (#255): one or two are held by CPython's critical section, three + # or more by every list's own lock (cgen.py, design/typedpython.md 4.3.6); the verifier refuses the + # latter when something in the call could wait (verify/array-locks). if fn.returns is None: raise Skip(line, "no return annotation") if isinstance(fn.returns, ast.Constant) and fn.returns.value is None: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 4d0e4f25f..3b0d9faa3 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -73,17 +73,31 @@ Python error set, nothing to undo), `-1` Python error set (CPython's exception t /* aliasing guard: 1 if any two of the n objects are the same object */ int tp_any_same(PyObject *const *objs, Py_ssize_t n); - /* Free-threaded CPython (#159): the wrapper holds the lists' critical section from before the - first enter to after the last exit, on every path (ok, error, deopt). tp_cs is zero-initialised - (`tp_cs c = {0};`). b == NULL locks one list, else both (PyCriticalSection2: ordered by - address, so two threads locking (a, b) and (b, a) cannot deadlock). More than two list - parameters are not compiled (frontend.Skip, verify/structure, CGenError): CPython has no - three-object section and a nested third would suspend the others. With a GIL both are no-ops. */ - typedef struct { int n; /* + a PyCriticalSection2 on free-threaded builds */ } tp_cs; + /* Free-threaded CPython (#159): the wrapper holds the lists' locks from before the first enter + to after the last exit, on every path (ok, error, deopt). tp_cs is zero-initialised + (`tp_cs c = {0};`). tp_cs_begin: b == NULL locks one list, else both (PyCriticalSection2: + ordered by address, so two threads locking (a, b) and (b, a) cannot deadlock). + tp_cs_begin_n (#255; agent proposal, not confirmed): three or more distinct exact lists; + sorts objs by address in place (objs stays the lock's, for the re-take after a poll) and holds + one list through a critical section and the others through their own mutex (ob_mutex), never + blocking while it holds a mutex directly; 0 = all held, 1 = not taken (nothing held: the + wrapper deopts, a poll leaves for the continuation) after TP_CS_ROUNDS (64) rounds, which + happens when two of the lists are held by this thread's own top-most PyCriticalSection2. With + a GIL all three are no-ops (tp_cs_begin_n returns 0). */ + typedef struct { int n, w; PyObject **objs; /* + a PyCriticalSection2 on free-threaded builds */ } tp_cs; void tp_cs_begin(tp_cs *c, PyObject *a, PyObject *b); + int tp_cs_begin_n(tp_cs *c, PyObject **objs, int n); void tp_cs_end(tp_cs *c); /* idempotent: a no-op when nothing is held */ + /* exit with three or more lists: the list's mutex is held directly, so the write-back is + PyList_SetItem's store without its critical section (which would wait for this thread) */ + int tp_f64_array_exit_held(tp_f64_array *a); + int tp_i64_array_exit_held(tp_i64_array *a); `tp_*_array_enter` / `_exit` read and write the list through borrowed references (`PyList_GET_ITEM`, -`PyList_SetItem`) and REQUIRE the caller to hold that critical section. Between enter and exit no +`PyList_SetItem`) and REQUIRE the caller to hold that critical section (`_exit_held`: the list's +mutex, held by `tp_cs_begin_n`). With three or more lists nothing between enter and exit may wait +(a detach or a blocking lock would keep the directly held mutexes while this thread waits): +verify/array-locks refuses an entry global or a redo Call anywhere in the call tree, and cgen +refuses them again (CGenError). Between enter and exit no other thread can change the list, so the whole call is one atomic step as far as lists go (an interleaving CPython can produce: each list operation takes the same lock). A long compiled array loop does not poll (below), so it delays other threads' stop-the-world requests (the free-threaded @@ -114,7 +128,8 @@ call by the export. Rules that go with it (verifier / front end): * Closed (#203) like a list: a function with a buffer runs no user code (`verify/array-open`). -* No limit on the number of buffer parameters (no critical section); lists keep the limit of two. +* No limit on the number of buffer parameters (no critical section), nor of list parameters since #255 + (three or more: `tp_cs_begin_n`, with the verify/array-locks condition). A buffer and a list cannot alias (different types). Two buffers may share memory (the same array passed twice): per-element access makes reads see earlier writes in program order, as in the interpreter, which a copy-in could not. @@ -234,7 +249,8 @@ Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryEr Thread state (#125): the wrapper calls `PyThreadState_GetUnchecked()` once and passes it as `tp_ts` to the impl, which passes it to `tp_enter_call`, `tp_depth_room`, `tp_traced` and to its direct compiled callees, so a compiled call and its C-to-C callees do one TLS lookup of the thread state per call. Likewise the per-thread state (call depth, poll countdown, snapshot depth) is one `__thread` struct -`tp_osr` (#152, OSR-style exit): `{ tp_cs *lock; PyObject *lists[2]; PyObject *result; }`. A snapshot-holding +`tp_osr` (#152, OSR-style exit): `{ tp_cs *lock; PyObject *const *lists; PyObject *result; }` (`lists`: the +wrapper's array of the list arguments in parameter order, any number since #255). A snapshot-holding function's wrapper sets `tp_t->osr` to its own context right before calling the impl; the impl takes it at entry and sets `tp_t->osr` back to NULL. The impl returns 2 when it left for the continuation, with `result` set (strong). @@ -483,6 +499,7 @@ static or module-state field must be added here): | `buf_types` (array.array) | module state | immutable after init | written by `tp_exec` (and `tp_clear`); read-only type pointers compared by identity | | buffer arguments of array parameters | caller's array.array memory | atomic | relaxed 8-byte `__atomic_load_n` / `__atomic_store_n` per element; the Py_buffer export is held for the call (open point on concurrent resize, #149) | | list arguments of array parameters | caller's lists | critical section | `tp_cs_begin` before the copy-in, `tp_cs_end` after the write-back, on every path | +| list arguments, three or more (#255) | caller's lists | each list's lock | `tp_cs_begin_n`: one list through a critical section, the others' `ob_mutex` tried without blocking (a taken one ends the round and is waited for through a critical section, nothing held); every lock held from copy-in to write-back, write-back by `_exit_held`; nothing in that window waits (verify/array-locks). Agent proposal, not confirmed (design/typedpython.md 4.3.6) | | module globals (class names, entry globals) | module dict | strong reads | `tp_global` and `tp_class_global_ok` use `PyDict_GetItemRef` | | a compiled class's instances | other threads' objects | CPython | `PyMember_GetOne` / `SetOne` are what the interpreter's member descriptors call (they take the object's lock on free-threaded builds); `tp_new_fixed` writes only its own fresh object | | `errno` (math helpers) | C library | thread-local | per thread by POSIX | @@ -496,7 +513,7 @@ a module is kept alive by an own reference while it is read; `PyEval_GetBuiltins `PyEval_GetFrameBuiltins` (a new reference). `tp_class_global_ok`: `PyDict_GetItemWithError` became `PyDict_GetItemRef` inside the dict's critical section. `tp_cg_redo`: the counter update is under the module dict's critical section. `PyList_GET_ITEM` in `tp_*_array_enter` stays a borrow: it is read -under the list's critical section the caller holds. Borrows that cannot race: `PyModule_GetDict` (the +under the list's critical section the caller holds (with three or more lists, the list's lock). Borrows that cannot race: `PyModule_GetDict` (the dict lives as long as the module, which the running function keeps alive), `PyTuple_GET_SIZE` of a kwnames tuple, `PyTuple` items. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 845f42ec9..4b909d9ae 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -112,11 +112,42 @@ static inline unsigned int tp_type_tag(const PyTypeObject *cls) #define TP_THREAD_LOCAL __thread #endif -/* Critical sections for array parameters (tp_cs_*): from before the copy-in to after the - * write-back, on every exit path. One list or two (CPython offers one- and two-object sections - * only; a function with more list parameters stays interpreted). With a GIL they are no-ops. */ +/* Locks for list parameters (tp_cs_*): from before the copy-in to after the write-back, on every + * exit path. With a GIL they are no-ops. + * + * One list or two: CPython's own critical section (PyCriticalSection / PyCriticalSection2, ordered + * by address). Three or more (#255, design/typedpython.md 4.3.6; agent proposal, not confirmed): + * CPython has no section for more than two objects, and a nested third may suspend the outer ones + * (Include/critical_section.h), so tp_cs_begin_n holds one list through a critical section and the + * others through their own mutex (ob_mutex): + * - it never blocks while it holds a mutex directly: each round begins a critical section on one + * list (which waits, if it must, the way CPython requires: this thread's own outer sections are + * suspended while it waits, and a list held by this thread's top-most section is not locked + * again), then only TRIES the other lists' mutexes (the compare-and-swap of PyMutex_LockFast + * and of the interpreter's LOCK_OBJECT). If one is taken it releases what it got, ends the + * section and begins the next round with a critical section on that list; + * - a list that a caller of this thread holds (list.sort holds its list while it calls the key) + * is either the round's own list (the top-most section holds it: not locked again) or is freed + * when the round's section waits (the wait suspends the caller's sections). Two lists held by + * one section of a caller (a PyCriticalSection2 that is top-most) can never both be had, so + * after TP_CS_ROUNDS rounds it gives up with nothing held and the call runs interpreted (a + * deopt, before any effect); contention alone ends a round only when another thread took a + * list between two of its attempts; + * - while they are held nothing may wait (the verifier's verify/array-locks: closed, no entry + * global, no redo Call): the section is not suspended (that needs a detach or a blocking nested + * section), and no thread can wait for this one while this one waits for it; + * - the write-back stores through tp__list_store_held, not PyList_SetItem, which would begin a + * critical section on a directly held list and wait for this very thread (listobject.c). + * Other threads' critical sections, and the interpreter's LOCK_OBJECT fast paths, on these lists + * see the same mutexes locked and wait (or take their locking path), as for a critical section. */ +#ifndef TP_CS_ROUNDS +#define TP_CS_ROUNDS 64 +#endif + typedef struct { - int n; /* 0 = not held, 1 or 2 objects locked */ + int n; /* 0 = not held; 1 or 2: a critical section; >= 3: n lists held */ + int w; /* n >= 3: the index in objs held through the critical section */ + PyObject **objs; /* n >= 3: the lists in address order (the wrapper's array, borrowed) */ #ifdef Py_GIL_DISABLED PyCriticalSection2 cs; #endif @@ -139,6 +170,66 @@ static inline void tp_cs_begin(tp_cs *c, PyObject *a, PyObject *b) #endif } +#ifdef Py_GIL_DISABLED +/* one non-blocking attempt: the first step of _PyMutex_LockTimed (Python/lock.c), which keeps the + * "has parked" bit; the bit values are those of Include/cpython/pylock.h */ +static inline int tp__mutex_try(PyMutex *m) +{ + uint8_t v = _Py_atomic_load_uint8_relaxed(&m->_bits); + while ((v & _Py_LOCKED) == 0) { + if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, (uint8_t)(v | _Py_LOCKED))) { + return 1; + } + } + return 0; +} +#endif + +/* n >= 3 distinct exact lists (the wrapper's aliasing guard ran): sorts objs by address in place and + * takes every list's lock. 0 = all held (end with tp_cs_end); 1 = not taken, nothing held. */ +static inline int tp_cs_begin_n(tp_cs *c, PyObject **objs, int n) +{ + int i, j; + for (i = 1; i < n; i++) { /* the order PyCriticalSection2 uses: lower address first */ + PyObject *o = objs[i]; + for (j = i; j > 0 && (uintptr_t)objs[j - 1] > (uintptr_t)o; j--) { + objs[j] = objs[j - 1]; + } + objs[j] = o; + } + c->objs = objs; +#ifdef Py_GIL_DISABLED + int w = 0; + for (int round = 0; round < TP_CS_ROUNDS; round++) { + int k; + PyCriticalSection_Begin((PyCriticalSection *)&c->cs, objs[w]); + for (k = 0; k < n; k++) { + if (k != w && !tp__mutex_try(&objs[k]->ob_mutex)) { + break; + } + } + if (k == n) { + c->w = w; + c->n = n; + return 0; + } + for (i = k - 1; i >= 0; i--) { + if (i != w) { + PyMutex_Unlock(&objs[i]->ob_mutex); + } + } + PyCriticalSection_End((PyCriticalSection *)&c->cs); + w = k; /* nothing held: the next round waits for the taken one */ + } + c->n = 0; + return 1; +#else + c->w = 0; + c->n = n; + return 0; +#endif +} + /* idempotent: a no-op when nothing is held */ static inline void tp_cs_end(tp_cs *c) { @@ -149,6 +240,16 @@ static inline void tp_cs_end(tp_cs *c) else if (c->n == 2) { PyCriticalSection2_End(&c->cs); } + else if (c->n > 2) { + /* the directly held mutexes first: ending the section may resume an outer one, which can + * wait, and nothing may be held directly then */ + for (int i = c->n - 1; i >= 0; i--) { + if (i != c->w) { + PyMutex_Unlock(&c->objs[i]->ob_mutex); + } + } + PyCriticalSection_End((PyCriticalSection *)&c->cs); + } #endif c->n = 0; } @@ -1000,7 +1101,8 @@ static inline int tp_unbox_bool(PyObject *o, int *out) * enter: rc 0 -> the struct holds the copy; rc 1 or -1 -> the struct is zeroed (so exit is a safe * no-op) and nothing was changed. `list` is borrowed: the caller keeps the argument alive. * - * Free-threaded (#159): the CALLER holds the list's critical section (tp_cs_begin) from before + * Free-threaded (#159): the CALLER holds the list's critical section (tp_cs_begin) or, with three + * or more lists, its mutex (tp_cs_begin_n, #255; then it ends with exit_held) from before * enter to after exit, on every exit path. enter and exit read and write the list's items through * borrowed references (PyList_GET_ITEM) and rely on that lock: no other thread can change the list * between the copy-in and the write-back, so the whole call is one atomic step as far as lists are @@ -1113,7 +1215,36 @@ static inline int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, return tp__array_slot(a->len, i, store, slot); } -static inline int tp_f64_array_exit(tp_f64_array *a) +/* The store of PyList_SetItem (listobject.c) for a list whose mutex this thread holds directly + * (tp_cs_begin_n): the same index check, release store and decref of the old item, without the + * critical section PyList_SetItem begins, which would wait for this very thread. The old item is an + * exact float or int (the copy-in checked, and the lock was held since), so its decref runs no code. + * Steals v. With a GIL: PyList_SetItem. */ +static inline int tp__list_store_held(PyObject *list, Py_ssize_t i, PyObject *v) +{ +#ifdef Py_GIL_DISABLED + PyListObject *l = (PyListObject *)list; + PyObject *old; + if (i < 0 || i >= Py_SIZE(l)) { + Py_DECREF(v); + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return -1; + } + old = l->ob_item[i]; + _Py_atomic_store_ptr_release(&l->ob_item[i], v); + Py_XDECREF(old); + return 0; +#else + return PyList_SetItem(list, i, v); +#endif +} + +static inline int tp__list_store(PyObject *list, Py_ssize_t i, PyObject *v, int held) +{ + return held ? tp__list_store_held(list, i, v) : PyList_SetItem(list, i, v); +} + +static inline int tp__f64_array_exit(tp_f64_array *a, int held) { int rc = 0; if (a->data != NULL && a->dirty != NULL && a->list != NULL) { @@ -1124,7 +1255,7 @@ static inline int tp_f64_array_exit(tp_f64_array *a) continue; } v = PyFloat_FromDouble(a->data[i]); - if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + if (v == NULL || tp__list_store(a->list, i, v, held) < 0) { rc = -1; /* SetItem consumed v on failure; the error is set */ break; } @@ -1139,7 +1270,7 @@ static inline int tp_f64_array_exit(tp_f64_array *a) return rc; } -static inline int tp_i64_array_exit(tp_i64_array *a) +static inline int tp__i64_array_exit(tp_i64_array *a, int held) { int rc = 0; if (a->data != NULL && a->dirty != NULL && a->list != NULL) { @@ -1150,7 +1281,7 @@ static inline int tp_i64_array_exit(tp_i64_array *a) continue; } v = PyLong_FromLongLong((long long)a->data[i]); - if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + if (v == NULL || tp__list_store(a->list, i, v, held) < 0) { rc = -1; break; } @@ -1165,6 +1296,13 @@ static inline int tp_i64_array_exit(tp_i64_array *a) return rc; } +/* exit: the list's critical section is held (one or two lists); exit_held: its mutex is held + * directly (three or more lists, tp_cs_begin_n) */ +static inline int tp_f64_array_exit(tp_f64_array *a) { return tp__f64_array_exit(a, 0); } +static inline int tp_i64_array_exit(tp_i64_array *a) { return tp__i64_array_exit(a, 0); } +static inline int tp_f64_array_exit_held(tp_f64_array *a) { return tp__f64_array_exit(a, 1); } +static inline int tp_i64_array_exit_held(tp_i64_array *a) { return tp__i64_array_exit(a, 1); } + /* ------------------------------------------------------------------------------------------ * Buffer arrays (issue #149; ir.ArrayParam.buffer): an exact array.array argument * is NOT copied. tp_buf_enter takes one Py_buffer export and tp_buf_exit releases it; the wrapper @@ -2866,7 +3004,7 @@ static inline PyObject *tp_list_from_i64(const int64_t *d, Py_ssize_t n) * returned 2; the wrapper then returns it, with its write-back already done. */ typedef struct tp_osr { tp_cs *lock; /* the wrapper's critical section, or NULL (no list parameter) */ - PyObject *lists[2]; /* the list arguments in critical-section order (borrowed), or NULL */ + PyObject *const *lists; /* the list arguments in parameter order (the wrapper's, borrowed), or NULL */ PyObject *result; /* strong, after an exit: what the continuation returned */ } tp_osr; diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index 6493c8f93..f76e6d820 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -37,14 +37,16 @@ verify/structure break/continue outside a loop, the ForRange var assigned in its body, a zero or non-constant step, duplicate names, bad declarations, an entry global that collides with a parameter/local/other entry global or - is assigned in the body, or more than two array parameters (the lists - are locked from copy-in to write-back: one- and two-object critical - sections only, #159) + is assigned in the body verify/proven-overflow `proven` is set on an op the verifier cannot prove stays inside i64 (or on a node `proven` does not apply to, or it is not a bool) verify/redo `Call.redo` where it is not legal (see below) verify/entry-global-open a function with entry_globals is not closed verify/array-open a function with an ArrayParam runs user code (#203) + verify/array-locks a function with three or more list ArrayParams (#255: it holds every + list's own lock from copy-in to write-back, outside CPython's + critical-section stack, so nothing in that window may wait) has an + entry global or a redo Call, itself or anywhere in its call tree verify/buffer a malformed `ArrayParam.buffer` (None, "array", "numpy", "memoryview"), or a CopyArray of a buffer array (#149) verify/par-loop a `ForRange.par` (SPEC N-15, #277) that the verifier does not recompute to @@ -308,6 +310,7 @@ REDO = "verify/redo" ENTRY_OPEN = "verify/entry-global-open" ARRAY_OPEN = "verify/array-open" +ARRAY_LOCKS = "verify/array-locks" BUFFER = "verify/buffer" CLASS = "verify/class" FIELD = "verify/field-unproved" @@ -433,6 +436,7 @@ def __init__(self, function: Function, signatures: dict[str, Function], self.entry: dict[str, Type] = {} # name -> type of every entry global self.opens: list[str] = [] # reasons the function is not closed self.slow_user: list[str] = [] # slow paths that can run user code (FieldGet, New) + self.redo_calls: list[str] = [] # callees of the legal redo Calls (#255: they can wait) def error(self, rule: str, message: str) -> None: d = Diagnostic(self.fn.name, rule, message, self.fn.source_line) @@ -469,10 +473,6 @@ def declarations(self) -> frozenset[str]: if p.name in params: self.error(STRUCTURE, f"parameter '{p.name}' appears twice") params.add(p.name) - if sum(type(p) is ArrayParam and p.buffer is None for p in fn.params) > 2: - self.error(STRUCTURE, "more than two array parameters: compiled code holds each list's " - "critical section from copy-in to write-back, and free-threaded " - "CPython has no section for more than two objects (#159)") eg = fn.entry_globals if not isinstance(eg, tuple): self.error(STRUCTURE, "entry_globals is not a tuple") @@ -1071,6 +1071,7 @@ def e_Call(self, e: Call, state: _State) -> ir.Expr: f"(the redone value may be a big int)") else: redo = True + self.redo_calls.append(e.function) if not redo: for p in callee.params: # The callee's entry guard `type(x) is C` may fail, even for an argument that is a @@ -2554,6 +2555,7 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: callees: dict[str, set[str]] = {} open_why: dict[str, str] = {} # a function that runs user code -> why slow_why: dict[str, str] = {} # a function whose slow paths can run user code -> why + wait_why: dict[str, str] = {} # a function that can wait (#255: entry global, redo) -> why for name, g in sigs.items(): c, rebuilt = _prove(g, sigs, classes) for d in c.diags: @@ -2564,6 +2566,12 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: open_why[name] = c.opens[0] if c.slow_user: slow_why[name] = c.slow_user[0] + if isinstance(g.entry_globals, tuple) and g.entry_globals: + wait_why[name] = (f"reads entry global '{getattr(g.entry_globals[0], 'name', '?')}' (the " + "read can take the module dict's critical section)") + elif c.redo_calls: + wait_why[name] = (f"has a redo Call of '{c.redo_calls[0]}' (the interpreted callee can " + "take critical sections)") # Closedness (entry_globals): a fixpoint over the call graph. changed = True @@ -2600,6 +2608,28 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: reject(name, g.source_line, ARRAY_OPEN, f"holds a list copy-in but can run user code: {array_why[name]}") + # Three or more list parameters (#255, design/typedpython.md 4.3.6): the wrapper holds every list's + # own lock (ob_mutex) from the copy-in to the write-back, outside CPython's critical-section stack. + # A detach or a blocking lock in that window would keep those locks while this thread waits, and a + # thread waiting for one of them while holding what this one waits for (PyCriticalSection2 keeps + # its first lock while it waits for the second) would never wake. So nothing in the window may + # wait: closed (above), and no entry global and no redo Call in the whole call tree. + changed = True + while changed: + changed = False + for name, cs in callees.items(): + if name not in wait_why: + bad = sorted(c for c in cs if c in wait_why) + if bad: + wait_why[name] = f"calls '{bad[0]}', which {wait_why[bad[0]]}" + changed = True + for name, g in sigs.items(): + n_lists = sum(isinstance(p, ArrayParam) and p.buffer is None for p in g.params) + if n_lists > 2 and name in wait_why: + reject(name, g.source_line, ARRAY_LOCKS, + f"three or more list parameters ({n_lists}) hold every list's lock from copy-in " + f"to write-back, so nothing in the call may wait, but it {wait_why[name]}") + # A call cycle (direct or mutual recursion) is legal: every compiled impl function counts its # frame at entry (tp_enter_call), so recursion ends in RecursionError where CPython raises it # and the C stack cannot overflow (issue #57). Purity and may_deopt flow through the cycle as From f64c1bc7e7864e16f65d8a852e64a939e049a36e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:16:19 +0900 Subject: [PATCH 35/61] Python3: public finalize() and flush(), one-argument eval(expression) (#339) finalize() runs Py_FinalizeEx, returns its status, and refuses to run off the thread that called initialize(). flush() flushes sys.stdout and sys.stderr. eval(expression) evaluates in __main__. SPEC L-1a; tests in Python3Test and the child-JVM InterpreterShutdownTest. --- README.md | 8 +- docs/SPEC.md | 12 ++ .../python/multiplatform/ffi/Python3.kt | 133 ++++++++++--- .../python/multiplatform/ffi/Python3Test.kt | 50 +++++ .../ffi/InterpreterShutdownTest.kt | 186 ++++++++++++++++++ 5 files changed, 360 insertions(+), 29 deletions(-) create mode 100644 python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt diff --git a/README.md b/README.md index f126a862d..523b8ce2d 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,16 @@ fun main() { val numbers = PyList.fromList(listOf(2L, 3L, 5L, 7L, 11L).map { PyInt.from(it) }) Python3.import("__main__").setAttr("kotlin_numbers", numbers) - val globals = Python3.import("__main__").dict - val result: PyObject = Python3.eval("sum(kotlin_numbers) * 2", 258 /* Py_eval_input */, globals, globals) + // One expression, evaluated in __main__. + val result: PyObject = Python3.eval("sum(kotlin_numbers) * 2") println("${result.Type.name}: $result") // int: 56 Python3.exec("print('hello from python')") // No close() anywhere: each wrapper releases its reference when it is collected. + + // Python buffers sys.stdout. finalize() flushes it and ends the interpreter (call it on the + // thread that called initialize()); Python3.flush() flushes without ending it. + Python3.finalize() } ``` diff --git a/docs/SPEC.md b/docs/SPEC.md index 90d07e494..a4485809d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -48,6 +48,18 @@ fail, then implement. `PyObject`, `Python3.import(name)` returns a `PyModule`. Python errors surface as `PyException` carrying the real Python exception (not a printed-and-cleared one). `Status: implemented`, `PM/commonTest/.../ffi/Python3Test.kt`, `SmokeTest.kt`, `VersionsTest.kt`. +- **L-1a** (issue #339) A consumer can end the interpreter and get the one-argument evaluation it + needs without reaching for C API constants. `Python3.eval(expression)` evaluates one expression in + `__main__`'s namespace (`Py_eval_input`) and returns a new `PyObject`. `Python3.flush()` flushes + `sys.stdout` and `sys.stderr` (a stream that is `None` or has no `flush` is skipped; an error in + `flush` surfaces as `PyException`), so output Python buffered is not lost when the process exits + without finalizing. `Python3.finalize()` is public: it runs `Py_FinalizeEx()` (which flushes the + standard streams itself), returns its status (0, or -1 when flushing buffered data failed), sets + `isInitialized` to false, and does nothing (returns 0) when the interpreter is not running. It must + run on the thread that called `initialize()`; from any other thread, or when this binding did not + start the interpreter, it throws `IllegalStateException` and leaves the interpreter running. Starting + the interpreter again in the same process after it is not supported. + `Status: implemented`, `PM/commonTest/.../ffi/Python3Test.kt`, `PM/desktopTest/.../ffi/InterpreterShutdownTest.kt`. - **L-2** `Python3.runMain` / `runApp` run a module or `-c` source with argv, map `sys.exit` status to a return value, restore argv, and do not finalize the shared interpreter. `Status: implemented`, `PM/commonTest/.../RunMainTest.kt`, `RunAppTest.kt`. diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index 5060b9469..cd6d591f9 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -136,51 +136,98 @@ object Python3 { } /** - * Finalize Python (Will not be able to re-initialize after this | Let this function be internal) + * Ends the interpreter (SPEC L-1a, issue #339) and returns `Py_FinalizeEx()`'s status: `0`, or + * `-1` when flushing buffered data failed. `Py_FinalizeEx()` flushes `sys.stdout` and + * `sys.stderr` itself, so output Python buffered reaches the process's streams. + * + * - Returns `0` and does nothing when the interpreter is not running. + * - Must be called on the thread that called [initialize]. From any other thread it throws + * [IllegalStateException] and the interpreter keeps running: `Py_FinalizeEx()` runs on the + * initialising thread's state, and adopting that state on another thread would leave two OS + * threads believing they own it. + * - Throws [IllegalStateException] as well when something other than [initialize] started the + * interpreter, since there is then no parked thread state to finalize on. + * - Afterwards [isInitialized] is false. Starting the interpreter again in the same process is + * not supported (extension modules and this binding's cached objects do not survive it). + * - Threads still inside [withPython] or [withGIL] are not waited for here; stop them first. */ - internal fun finalize(silent: Boolean = false) { - if (!isInitialized) return - // Py_Finalize() requires the GIL. Reclaim the state parked by initialize(). - mainThreadState?.let { - PyEval_RestoreThread(it) - mainThreadState = null + fun finalize(silent: Boolean = false): Int { + if (!isInitialized) return 0 + val parked = mainThreadState + ?: throw IllegalStateException( + "Python3.finalize(): the interpreter was not started by Python3.initialize(), so there is no thread state to finalize on" + ) + // PyGILState_GetThisThreadState needs no GIL and returns the state CPython associates with + // the calling OS thread. On the initialising thread that is the state parked by + // initialize(); on any other thread it is null or that thread's own state. + if (PyGILState_GetThisThreadState() != parked) { + throw IllegalStateException( + "Python3.finalize() must be called on the thread that called Python3.initialize()" + ) } - // Lowered BEFORE Py_Finalize(), not after. Cleaner threads consult this flag to decide + // Py_FinalizeEx() requires the GIL. Reclaim the state parked by initialize(). + PyEval_RestoreThread(parked) + mainThreadState = null + // Lowered BEFORE Py_FinalizeEx(), not after. Cleaner threads consult this flag to decide // whether releasing a reference is still legal, and finalization is exactly the window // where it stops being legal -- leaving the flag up until afterwards let a cleaner pass - // the check and call Py_DecRef while Py_Finalize() was tearing the interpreter down, + // the check and call Py_DecRef while finalization was tearing the interpreter down, // which crashed the process inside _PyObject_ClearFreeLists. // // With the flag down first, a cleaner arriving from here on returns without touching the - // C API at all. One already inside its GIL scope still holds the GIL, and Py_Finalize() + // C API at all. One already inside its GIL scope still holds the GIL, and finalization // waits for it, so that case finishes safely before teardown begins. isInitialized = false - // The checkpoint function dies with the interpreter; drop the pointer so a later - // initialize() rebuilds it instead of calling through a dangling one. Same for the - // runMain/runApp helpers, which are cached the same way and would dangle the same way. + // The checkpoint function dies with the interpreter; drop the pointer so nothing calls + // through a dangling one. Same for the runMain/runApp helpers, which are cached the same way. checkpointCallable = null builtinCompiledInstalled = false runModuleCallable = null runTopLevelCallable = null runFileCallable = null - memScoped { - Py_Finalize() - // No error message is printable here, and that is a property of the C API rather than - // an omission. `Py_Finalize()` returns void; by the time it returns there is no - // interpreter left to hold an error indicator, so `PyErr_*` cannot be consulted. The - // only status available at all is `Py_FinalizeEx()`'s `int` -- 0, or -1 when flushing - // buffered data failed -- which carries no message either. Switching to it is a real - // (small) improvement and is recorded in ROADMAP §12 rather than done here, because - // finalization is untested: the only caller is `artMain/JniExport.kt`, and a test - // that exercises it destroys the interpreter the rest of the suite shares. - if (Py_IsInitialized() != 0) { - isInitialized = true - throw IllegalStateException("Python finalization failed") + // No error message is available after this, and that is a property of the C API: once + // Py_FinalizeEx() returns there is no interpreter left to hold an error indicator. Its int + // status is the whole report. + val status = Py_FinalizeEx() + if (Py_IsInitialized() != 0) { + isInitialized = true + throw IllegalStateException("Python finalization failed") + } + if (!silent) println("INFO: Python finalized (status $status)") + return status + } + + /** + * Flushes `sys.stdout` and `sys.stderr` (SPEC L-1a, issue #339). Python buffers its standard + * streams, and a process that exits without [finalize] loses what is still in the buffer. A + * stream that is `None` or has no `flush` is skipped; an exception raised by `flush` surfaces + * as a [PyException]. + * + * Runs in a fresh dictionary, so it leaves no names behind in `__main__`. + */ + fun flush() { + withPython { + // PyDict_New: new reference, released below. + val globals = PyDict_New() ?: throw pyErrorOrGeneric("Failed to allocate the flush namespace") + try { + // PyRun_String: new reference (None) on success, null + exception set on failure. + val result = PyRun_String(FLUSH_SOURCE, PY_FILE_INPUT, globals, globals) + ?: throw pyErrorOrGeneric("Flushing Python's standard streams failed") + Py_DecRef(result) + } finally { + Py_DecRef(globals) } - if (!silent) println("INFO: Python finalized successfully!") } } + private const val FLUSH_SOURCE = """ +import sys +for _s in (sys.stdout, sys.stderr): + _f = getattr(_s, "flush", None) + if _f is not None: + _f() +""" + /** * Runs the module [moduleName] as `__main__`, the way `python -m ` does, in an * interpreter that is **still alive when this returns**. @@ -725,6 +772,9 @@ def __pmp_run_file__(path, argv): /** `Py_file_input`, the compiler-mode token for a sequence of statements (as opposed to a single expression). */ private const val PY_FILE_INPUT: Int = 257 + /** `Py_eval_input`, the start symbol of the one-argument [eval]. */ + private const val PY_EVAL_INPUT: Int = 258 + /** * Outermost [withGIL] scopes between automatic checkpoints when they are enabled. * @@ -946,6 +996,35 @@ def __pmp_run_file__(path, argv): } } + /** + * Evaluates one expression in `__main__`'s namespace and returns the result as a new + * [PyObject] (SPEC L-1a, issue #339). The common case of the four-argument form, with + * `Py_eval_input`: a statement is a `SyntaxError`, raised as a [PyException]. Use [exec] for + * statements. + */ + fun eval(expression: String): PyObject { + return withPython { + // PyImport_AddModuleRef: new/strong reference to the __main__ module. + val modulePointer = PyImport_AddModuleRef("__main__") + ?: throw pyErrorOrGeneric("Failed to access the __main__ module") + try { + // PyObject_GetAttrString: new reference to __main__'s globals dict. + val globalsPointer = PyObject_GetAttrString(modulePointer, "__dict__") + ?: throw pyErrorOrGeneric("Failed to access __main__.__dict__") + try { + // PyRun_String: new reference on success, adopted by the wrapper. + val result = PyRun_String(expression, PY_EVAL_INPUT, globalsPointer, globalsPointer) + ?: throw pyErrorOrGeneric("Python eval failed") + PyObject(result, false) + } finally { + Py_DecRef(globalsPointer) + } + } finally { + Py_DecRef(modulePointer) + } + } + } + /** * Evaluate Python script */ diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt index 480ae1dee..d8f7a4a80 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/Python3Test.kt @@ -58,6 +58,56 @@ class Python3Test { } } + // SPEC L-1a (#339): the one-argument eval. Red before #339 as a compile error (no such + // overload), which cannot be mistaken for a behavioural regression. + + @Test + fun evalWithOneArgumentReadsMainNamespace() = PythonTestFixture.withInterpreter { + Python3.exec("x_for_one_arg_eval = 20") + val result = Python3.eval("x_for_one_arg_eval + 22") + assertEquals("42", result.toString()) + } + + @Test + fun evalWithOneArgumentRejectsAStatement() = PythonTestFixture.withInterpreter { + // Py_eval_input, not Py_file_input: an assignment is a SyntaxError, not a silent None. + assertFailsWith { Python3.eval("y_for_one_arg_eval = 1") } + } + + // SPEC L-1a (#339): flush(). The end-to-end proof that buffered output reaches the process's + // stdout is desktopTest's InterpreterShutdownTest; these pin the edge cases in-process. + + @Test + fun flushSkipsAMissingStream() = PythonTestFixture.withInterpreter { + Python3.exec("import sys as _pm_t_sys; _pm_t_saved = _pm_t_sys.stdout; _pm_t_sys.stdout = None") + try { + Python3.flush() + } finally { + Python3.exec("_pm_t_sys.stdout = _pm_t_saved; del _pm_t_saved, _pm_t_sys") + } + } + + @Test + fun flushReportsAFailingStreamAsPyException() = PythonTestFixture.withInterpreter { + Python3.exec( + """ + import sys as _pm_t_sys + class _PmFailingStream: + def write(self, s): return len(s) + def flush(self): raise ValueError("flush failed on purpose") + _pm_t_saved = _pm_t_sys.stderr + _pm_t_sys.stderr = _PmFailingStream() + """.trimIndent() + ) + try { + val error = assertFailsWith { Python3.flush() } + // The real ValueError, not a generic failure. + assertTrue("flush failed on purpose" in error.value.toString(), "got: ${error.value} / ${error.message}") + } finally { + Python3.exec("_pm_t_sys.stderr = _pm_t_saved; del _pm_t_saved, _pm_t_sys, _PmFailingStream") + } + } + @Test fun importReturnsAUsableModule() = PythonTestFixture.withInterpreter { val math = Python3.import("math") diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt new file mode 100644 index 000000000..9d9eb1967 --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/InterpreterShutdownTest.kt @@ -0,0 +1,186 @@ +package python.multiplatform.ffi + +import java.io.File +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import kotlin.system.exitProcess +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * SPEC L-1a (issue #339), end to end: what a consumer app sees on its process stdout when it ends + * the interpreter with the public `Python3.finalize()` or flushes it with `Python3.flush()`. + * + * Each case is a child JVM. Finalizing destroys the interpreter every other test in this JVM shares, + * and the point is the process's real stdout (a file here, so Python's `sys.stdout` is block + * buffered exactly as in a packaged app whose output is redirected). The child is launched the way + * `PackagedPythonHomeLaunchTest`'s environment-`PYTHONHOME` guard launches its probe. + * + * Red before #339: [ShutdownProbe] calls `Python3.flush()`, which did not exist, so this file did not + * compile (a compile error, not a behavioural failure). [withoutFlushTheBufferedLineIsLost] is the + * control that shows the test can fail: the same probe without the flush loses the line. + */ +class InterpreterShutdownTest { + + private val scratchDirs = mutableListOf() + + @AfterTest + fun cleanup() { + scratchDirs.forEach { dir -> + if (Files.exists(dir.toPath(), java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + Files.walk(dir.toPath()).use { paths -> + paths.sorted(Comparator.reverseOrder()).forEach { Files.delete(it) } + } + } + } + scratchDirs.clear() + } + + /** Under `build/`, not the system temporary directory (AGENTS.md rule 2). */ + private fun newScratchDir(): File = + File("build/tmp/interpreterShutdownTest/${java.util.UUID.randomUUID()}").absoluteFile + .also { it.mkdirs(); scratchDirs += it } + + private class Outcome(val exitCode: Int, val output: String) + + private fun launch(scenario: String): Outcome { + val home = assertNotNull(System.getenv("PYTHONHOME"), "PYTHONHOME is unset for desktopTest") + val workDir = newScratchDir() + val log = File(workDir, "probe.log") + val java = File(System.getProperty("java.home"), "bin/java").path + val command = listOf( + java, + "--enable-preview", + "--enable-native-access=ALL-UNNAMED", + "-Djava.library.path=.", + "-cp", + System.getProperty("java.class.path"), + ShutdownProbe::class.java.name, + scenario, + ) + val builder = ProcessBuilder(command) + .directory(workDir) + .redirectErrorStream(true) + .redirectOutput(log) + builder.environment().remove("PYTHON_MULTIPLATFORM_LIBPYTHON") + // Python would line-buffer or not buffer at all with this set, and the test would prove nothing. + builder.environment().remove("PYTHONUNBUFFERED") + builder.environment()["PYTHONHOME"] = home + val process = builder.start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + return Outcome(-1, "timed out after 120 s\n" + log.readText()) + } + return Outcome(process.exitValue(), log.readText()) + } + + @Test + fun finalizeFlushesBufferedOutputAndReportsStatus() { + val outcome = launch("finalize") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue(ShutdownProbe.BUFFERED_LINE in outcome.output, "Python's buffered stdout was lost:\n${outcome.output}") + assertTrue("FINALIZED status=0 initialized=false" in outcome.output, outcome.output) + } + + @Test + fun flushWithoutFinalizeKeepsBufferedOutput() { + val outcome = launch("flush") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue(ShutdownProbe.BUFFERED_LINE in outcome.output, "Python's buffered stdout was lost:\n${outcome.output}") + } + + @Test + fun withoutFlushTheBufferedLineIsLost() { + // Control: if this ever finds the line, stdout is not buffered in the child and the two + // tests above no longer prove anything. + val outcome = launch("noflush") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("NOFLUSH_DONE" in outcome.output, outcome.output) + assertFalse(ShutdownProbe.BUFFERED_LINE in outcome.output, "stdout was not buffered:\n${outcome.output}") + } + + @Test + fun finalizeFromAnotherThreadIsRefusedAndTheInterpreterSurvives() { + val outcome = launch("otherthread") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("OTHER_THREAD_REFUSED" in outcome.output, outcome.output) + assertTrue("STILL_RUNNING 42" in outcome.output, outcome.output) + assertTrue("FINALIZED status=0 initialized=false" in outcome.output, outcome.output) + } + + @Test + fun finalizeWhenNotRunningReturnsZero() { + val outcome = launch("notrunning") + assertEquals(0, outcome.exitCode, outcome.output) + assertTrue("NOT_RUNNING status=0" in outcome.output, outcome.output) + } +} + +/** The child's `main`; `args[0]` names the scenario. Kotlin output always goes through `println` + flush. */ +object ShutdownProbe { + const val BUFFERED_LINE = "PMP_BUFFERED_LINE_339" + + private fun writeBuffered() { + // No flush=True and no newline-triggered flush: the stream is a file, so it is block buffered. + Python3.exec("import sys\nsys.stdout.write('$BUFFERED_LINE\\n')") + } + + private fun say(line: String) { + println(line) + System.out.flush() + } + + @JvmStatic + fun main(args: Array) { + when (args.single()) { + "finalize" -> { + Python3.initialize(silent = true) + writeBuffered() + val status = Python3.finalize(silent = true) + say("FINALIZED status=$status initialized=${Python3.isInitialized}") + exitProcess(0) + } + "flush" -> { + Python3.initialize(silent = true) + writeBuffered() + Python3.flush() + // halt: no shutdown hooks, nothing else gets a chance to flush on our behalf. + Runtime.getRuntime().halt(0) + } + "noflush" -> { + Python3.initialize(silent = true) + writeBuffered() + say("NOFLUSH_DONE") + Runtime.getRuntime().halt(0) + } + "otherthread" -> { + Python3.initialize(silent = true) + var refused = false + val thread = Thread { + try { + Python3.finalize(silent = true) + } catch (e: IllegalStateException) { + refused = true + } + } + thread.start() + thread.join() + if (refused) say("OTHER_THREAD_REFUSED") + say("STILL_RUNNING ${Python3.eval("6 * 7")}") + val status = Python3.finalize(silent = true) + say("FINALIZED status=$status initialized=${Python3.isInitialized}") + exitProcess(0) + } + "notrunning" -> { + val status = Python3.finalize(silent = true) + say("NOT_RUNNING status=$status") + exitProcess(0) + } + else -> exitProcess(2) + } + } +} From f16e5f2f87f53f3d118ed3b5f4503b5e3eb2e813 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:19:11 +0900 Subject: [PATCH 36/61] Compose host: add macosArm64 and linuxX64 Kotlin/Native desktop targets (#386) Part of #191 Phase B. Same intermediate source set as #370 (commonMain -> desktopNativeMain -> {macosArm64Main, linuxX64Main}); the host is all commonMain. Compose gates macosArm64 behind org.jetbrains.compose.experimental.macos.enabled. linuxX64 resolves only on the fork (upstream publishes no ui-linuxx64). The desktop-native-linux job also compiles the host for linuxX64. --- .github/workflows/desktop.yml | 5 +++++ docs/SPEC.md | 5 ++++- gradle.properties | 2 ++ python-multiplatform-compose/build.gradle.kts | 17 +++++++++++++++-- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 7d6003ba4..907744729 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -76,6 +76,11 @@ jobs: - name: Run Linux x86_64 Kotlin/Native Tests run: ./gradlew :python-multiplatform:linuxX64Test --rerun --console=plain + # The Compose host's linuxX64 target (#386): compiled here, since only the fork publishes the + # Compose UI klibs for linuxX64. + - name: Compile the Compose host for Linux x86_64 Kotlin/Native + run: ./gradlew :python-multiplatform-compose:compileKotlinLinuxX64 --rerun --console=plain + # The dev agent's protocol on the other desktop OSes (#238). Its socket options differ per platform # (SO_EXCLUSIVEADDRUSE on Windows, SO_REUSEADDR on POSIX), and the port-takeover and restart tests are # what check them. Stdlib only: no JDK, no Gradle. diff --git a/docs/SPEC.md b/docs/SPEC.md index 90d07e494..b2152da6a 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -542,7 +542,10 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. the next frame with no host call; Python callables a root passed into composables are released when that root is replaced or the composition is disposed. The entry point names no library. `Status: implemented` on desktop, `python-multiplatform-compose/fixtures/compose/.../PythonAppViewRenderTest.kt`; the module - compiles for Android, nothing runs there yet. + compiles for Android, nothing runs there yet. It also compiles for the Kotlin/Native desktop targets + `macosArm64` and `linuxX64` (#386, `commonMain -> desktopNativeMain -> {macosArm64Main, linuxX64Main}`; + `linuxX64` exists only on the fork, upstream Compose publishes no `ui` klib for it); nothing runs there yet + (#191 Phase B: the klib walker and a K/N app). - **B-9** A Python root keeps state across the recreation of its composition with `python.multiplatform.compose.rememberSaveableWrapper(initial)` (the pycomposeui function of that name), callable from Python inside a composition. A Python `int` (32 bit) becomes `rememberSaveable { mutableIntStateOf }`, diff --git a/gradle.properties b/gradle.properties index 069e2f372..75120a9bd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,6 +21,8 @@ kotlin.native.binary.sourceInfoType=libbacktrace #Compose org.jetbrains.compose.experimental.jscanvas.enabled=true org.jetbrains.compose.experimental.uikit.enabled=true +# Compose gates its macosArm64 target behind this flag; :python-multiplatform-compose has it since #386. +org.jetbrains.compose.experimental.macos.enabled=true # org.thisisthepy.compose:compose-gradle-plugin:1.11.1-ext.3's Maven Central POM marks # hot-reload-gradle-plugin as optional (shadow configuration), so Gradle does not place it on the # plugin classpath unless disabled or declared separately. diff --git a/python-multiplatform-compose/build.gradle.kts b/python-multiplatform-compose/build.gradle.kts index 353384d43..11cb6e6fc 100644 --- a/python-multiplatform-compose/build.gradle.kts +++ b/python-multiplatform-compose/build.gradle.kts @@ -7,8 +7,8 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget * composable. * * A separate module so that `:python-multiplatform` itself never depends on Compose (same naming as - * `-ksp` and `-gradle-plugin`). JVM targets first -- desktop and Android share the code, which is all - * in `commonMain`; iOS and wasm come later. + * `-ksp` and `-gradle-plugin`). Desktop JVM, Android and Kotlin/Native desktop (macosArm64, linuxX64) + * share the code, which is all in `commonMain`; iOS and wasm come later. */ plugins { alias(libs.plugins.kotlin.multiplatform) @@ -35,6 +35,12 @@ kotlin { } } + // Kotlin/Native desktop (#191 Phase B, #386), the targets `:python-multiplatform` has since #370. + // Upstream JetBrains Compose publishes no `ui` klib for linuxX64; the fork this repository builds + // on publishes `runtime`, `runtime-saveable` and `ui` for both targets. + macosArm64() + linuxX64() + sourceSets { commonMain.dependencies { api(projects.pythonMultiplatform) @@ -42,6 +48,13 @@ kotlin { api(compose.ui) api(compose.runtimeSaveable) } + // Same shape as the library's: commonMain -> desktopNativeMain -> {macosArm64Main, linuxX64Main}. + // Empty today (the host is all commonMain); it is where K/N-desktop-only host code goes. + val desktopNativeMain by creating { + dependsOn(commonMain.get()) + } + getByName("macosArm64Main").dependsOn(desktopNativeMain) + getByName("linuxX64Main").dependsOn(desktopNativeMain) } } From 99f7cc1e8dc2f74de140e9baacc1f90cbfad1290 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:22:21 +0900 Subject: [PATCH 37/61] TypedPython CI: dispatch input race_mutant runs the race mutant tests (#394) --- .github/workflows/typedpython.yml | 11 +++++++++++ python-multiplatform-ksp/pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml index 7c0cee003..70c97fee9 100644 --- a/.github/workflows/typedpython.yml +++ b/.github/workflows/typedpython.yml @@ -9,6 +9,13 @@ on: branches: [ "develop", "main" ] paths: [ "python-multiplatform-ksp/**", "python-multiplatform-ksp/fixtures/app/**", ".github/workflows/typedpython.yml" ] workflow_dispatch: + inputs: + race_mutant: + # The race_mutant tests show that a concurrency test catches a planted mutant; each depends + # on the race firing, so they run only when asked (leader decision 2026-11-13, #394). + description: "Also run the race_mutant tests (a missed race fails the step)" + type: boolean + default: false concurrency: # A run in progress finishes; a newer push replaces only the queued run (frequent merges # to develop otherwise cancelled every run before it reported). @@ -50,6 +57,10 @@ jobs: - name: Behaviour tests run: ${{ matrix.venv_python }} -m pytest -q -m "not measurement and not race_mutant" + - name: Race mutants (dispatch input race_mutant) + if: ${{ github.event_name == 'workflow_dispatch' && inputs.race_mutant }} + run: ${{ matrix.venv_python }} -m pytest -q -rA -m race_mutant + - name: Measurements (recorded, sanity bounds only) # A shared runner is not an idle machine (AGENTS.md §8): read ratios, not absolute times. run: ${{ matrix.venv_python }} -m pytest -q -s -m measurement diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index 487554414..1f50ef871 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -38,5 +38,5 @@ pythonpath = ["src/main/python"] addopts = "--basetemp=.pytest_tmp" markers = [ "measurement: records cost; asserts only sanity bounds", - "race_mutant: shows a concurrency test catches a mutant; depends on the race firing, so local only (not in CI)", + "race_mutant: shows a concurrency test catches a mutant; depends on the race firing, so not in push CI; run with the TypedPython CI dispatch input race_mutant (#394)", ] From a028e1ca08459bd615d598b8ea94e547a28bb2d7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:25:03 +0900 Subject: [PATCH 38/61] Plugin: iOS consumer wiring is 3.15t only, no freeThreaded switch (#328) Leader decision 2026-11-13 on #158 (3.15t is the only build): the python.org GIL iOS archive path and the ios.freeThreaded property are removed from the plugin; the free-threaded pin is required at plugin build time. --- docs/SPEC.md | 28 ++-- .../build.gradle.kts | 65 ++------ .../multiplatform/gradle/IosConsumerWiring.kt | 43 ++---- .../multiplatform/gradle/IosPythonHome.kt | 41 ++--- .../gradle/IosConsumerWiringTest.kt | 30 +--- .../gradle/IosPythonHomeLayoutTest.kt | 145 +++++------------- 6 files changed, 102 insertions(+), 250 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 537b64f3a..0dbebf053 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -661,20 +661,20 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. python.org and python-build-standalone publish free-threaded prebuilts for desktop only; for Android and iOS this project builds its own (#163, release `cpython-3.15.0t-mobile`), see T-2a. `Py_LIMITED_API` is not defined, so `abi3t` is not a blocker. -- **T-2a** Free-threaded CPython on mobile through the Gradle plugin (#328 step 1; direction: the - maintainer's #158 decision of 2026-10-04, "Free-threaded is the default and the only build", baseline - 3.15t). The plugin's iOS consumer wiring (#90, L-11) can acquire the project's own free-threaded - `Python.xcframework`, `python-t-iOS-XCframework.tar.gz` from the GitHub release - `cpython-t-mobile` of `thisisthepy/python-multiplatform`, verified against the SHA-256 pinned in - `python-checksums.properties` under `ios--freethreaded` and baked into the plugin at plugin build - time (a mismatch, or a missing pin, fails `downloadPython_ios`). It is selected per consumer with - `pythonBindings { ios { freeThreaded.set(true) } }`; the default stays `false`, the python.org GIL - archive and `lib/python/`, until #328 step 3 switches the mobile default (the separate switch, - and its default, are an agent proposal, not confirmed). With it set, every per-slice prefix holds - `lib/pythont/`: the shared `Python.xcframework/lib/pythont/` merged with - `/lib-/pythont/`, with L-11's exclusions (so no `libpythont.dylib`), and the - staging task fails unless `lib/pythont/os.py` is staged. Android: the plugin stages nothing for - Android (libpython and the stdlib travel inside the library's AAR, L-5); the Android archive's pin +- **T-2a** Free-threaded CPython on mobile through the Gradle plugin (#328 step 1; owner decision #158 of + 2026-10-04, "Free-threaded is the default and the only build", baseline 3.15t; leader decision of + 2026-11-13 on #328: no opt-in `pythonBindings.ios.freeThreaded` flag, 3.15t is the only build). There + is no switch and no GIL iOS archive in the plugin. The plugin's iOS consumer wiring (#90, L-11) always + acquires the project's own free-threaded `Python.xcframework`, `python-t-iOS-XCframework.tar.gz` + from the GitHub release `cpython-t-mobile` of `thisisthepy/python-multiplatform`, verified + against the SHA-256 pinned in `python-checksums.properties` under `ios--freethreaded` and baked + into the plugin at plugin build time (a missing pin fails the plugin build, a mismatch fails + `downloadPython_ios`). Every per-slice prefix holds `lib/pythont/`: the shared + `Python.xcframework/lib/pythont/` merged with `/lib-/pythont/`, with L-11's + exclusions (so no `libpythont.dylib`), and the staging task fails unless `lib/pythont/os.py` is + staged. The library's own iOS klib must also be built against the 3.15t headers before a consumer app + runs on this framework (#328 step 2, done separately). Android: the plugin stages nothing for Android + (libpython and the stdlib travel inside the library's AAR, L-5); the Android archive's pin (`android-aarch64--freethreaded`) is recorded in `python-checksums.properties`, and acquiring it is a library-build change left to #328 steps 2 and 3. `Status: partial` (unit tests only, `GP/IosConsumerWiringTest.kt`, `GP/IosPythonHomeLayoutTest.kt`; no simulator run, and the library's own diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index f7229c7f0..728ac2a98 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -260,44 +260,21 @@ val generateCoordinates = tasks.register("generateCoordinates") { val klibWorkerValue = klibWorkerCoordinates - // The CPython iOS support archive the library build downloads, derived with the same rule as - // `python-multiplatform/build.gradle.kts` (`downloadPython_ios`): python.org from 3.15, BeeWare - // before. The SHA-256 is read from the root `python-checksums.properties` under the library's - // own lock key and baked into the plugin, so a consumer verifies against the pin this plugin - // was built with and needs no lockfile of its own. - val iosVersionParts = pythonVersionValue.split(".") - val iosFromPythonOrgValue = iosVersionParts[0].toInt() > 3 || (iosVersionParts[0].toInt() == 3 && iosVersionParts[1].toInt() >= 15) - val iosLibVersion = "${iosVersionParts[0]}.${iosVersionParts[1]}" - val iosReleaseDir = "$iosLibVersion.${iosVersionParts.getOrNull(2)?.takeWhile { it.isDigit() } ?: "0"}" - val appleSupportBuild = rootProperty("pythonAppleSupportBuild", "b11") - val iosArchiveNameValue = if (iosFromPythonOrgValue) { - "python-$pythonVersionValue-iOS-XCframework.tar.gz" - } else { - "Python-$iosLibVersion-iOS-support.$appleSupportBuild.tar.gz" - } - val iosUrlValue = if (iosFromPythonOrgValue) { - "https://www.python.org/ftp/python/$iosReleaseDir/$iosArchiveNameValue" - } else { - "https://github.com/beeware/Python-Apple-support/releases/download/$iosLibVersion-$appleSupportBuild/$iosArchiveNameValue" - } - val iosLockKeyValue = if (iosFromPythonOrgValue) "ios-$pythonVersionValue-pythonorg" else "ios-$iosLibVersion-$appleSupportBuild" + // The iOS support archive is the project's own free-threaded 3.15t build (#163, #328; owner + // decision #158: free-threaded 3.15t is the only build, no GIL archive is kept). Release + // `cpython-t-mobile` of thisisthepy/python-multiplatform, pinned under + // `ios--freethreaded` in the root `python-checksums.properties` and baked into the plugin, + // so a consumer verifies against the pin this plugin was built with and needs no lockfile of its + // own. The 3.15.0 value is the release's SHA256SUMS line, equal to the GitHub asset digest and to + // `shasum -a 256` of the downloaded archive (checked 2026-11-13). val checksumsFile = project.rootDir.resolve("../python-checksums.properties") - val iosSha256Value = Properties().apply { - if (checksumsFile.isFile) checksumsFile.inputStream().use { load(it) } - }.getProperty(iosLockKeyValue) - ?: throw GradleException( - "python-checksums.properties (${checksumsFile.path}) has no '$iosLockKeyValue'; the plugin bakes the iOS support archive's SHA-256 from it.", - ) - - // The project's own free-threaded iOS build (#163, #328 step 1): release `cpython-t-mobile` - // of thisisthepy/python-multiplatform. Pinned under `ios--freethreaded`; the 3.15.0 value - // is the release's SHA256SUMS line, equal to the GitHub asset digest and to `shasum -a 256` of the - // downloaded archive (checked 2026-11-13). Optional: a missing pin bakes "" and only a consumer - // that selects the free-threaded archive fails, in `downloadPython_ios`, naming the key. val iosFreeThreadedLockKeyValue = "ios-$pythonVersionValue-freethreaded" val iosFreeThreadedSha256Value = Properties().apply { if (checksumsFile.isFile) checksumsFile.inputStream().use { load(it) } - }.getProperty(iosFreeThreadedLockKeyValue).orEmpty() + }.getProperty(iosFreeThreadedLockKeyValue) + ?: throw GradleException( + "python-checksums.properties (${checksumsFile.path}) has no '$iosFreeThreadedLockKeyValue'; the plugin bakes the iOS support archive's SHA-256 from it.", + ) inputs.property("coordinates", coordinates) inputs.property("klibWorkerCoordinates", klibWorkerValue) @@ -305,10 +282,6 @@ val generateCoordinates = tasks.register("generateCoordinates") { inputs.property("pbsRelease", pbsReleaseValue) inputs.property("pythonFreeThreaded", freeThreadedValue) inputs.property("wasmRuntimeCoordinates", wasmRuntimeCoordinatesValue) - inputs.property("iosUrl", iosUrlValue) - inputs.property("iosArchiveName", iosArchiveNameValue) - inputs.property("iosLockKey", iosLockKeyValue) - inputs.property("iosSha256", iosSha256Value) inputs.property("iosFreeThreadedLockKey", iosFreeThreadedLockKeyValue) inputs.property("iosFreeThreadedSha256", iosFreeThreadedSha256Value) outputs.dir(outputDir) @@ -328,22 +301,10 @@ val generateCoordinates = tasks.register("generateCoordinates") { /** The python-build-standalone release tag that published it. */ internal const val DEFAULT_PBS_RELEASE: String = "$pbsReleaseValue" - /** The pinned CPython iOS support archive: where it is downloaded from. */ - internal const val DEFAULT_IOS_ARCHIVE_URL: String = "$iosUrlValue" - - /** Its file name, which is also the file's name under the consumer's `build/python-standalone/`. */ - internal const val DEFAULT_IOS_ARCHIVE_NAME: String = "$iosArchiveNameValue" - - /** The key it has in the root `python-checksums.properties`, kept for messages. */ - internal const val DEFAULT_IOS_LOCK_KEY: String = "$iosLockKeyValue" - - /** Its SHA-256, read from `python-checksums.properties` when this plugin was built. */ - internal const val DEFAULT_IOS_ARCHIVE_SHA256: String = "$iosSha256Value" - - /** The key of the free-threaded iOS archive (SPEC T-2a) in `python-checksums.properties`. */ + /** The key of the iOS support archive (the 3.15t build, SPEC T-2a) in `python-checksums.properties`. */ internal const val DEFAULT_IOS_FREE_THREADED_LOCK_KEY: String = "$iosFreeThreadedLockKeyValue" - /** Its SHA-256, or "" when `python-checksums.properties` had no such key at plugin build time. */ + /** Its SHA-256, read from `python-checksums.properties` when this plugin was built. */ internal const val DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256: String = "$iosFreeThreadedSha256Value" internal const val DEFAULT_PYTHON_FREE_THREADED: Boolean = $freeThreadedValue diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt index ed615f1c0..311cf9f24 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt @@ -33,18 +33,6 @@ interface PythonBindingsIosExtension { * `Python.framework` some other way. */ val linkPythonFramework: Property - - /** - * Whether to acquire the free-threaded CPython (`3.15t`) `Python.xcframework` this project builds - * and publishes on its own GitHub release, instead of python.org's GIL one (SPEC T-2a, #328 step 1). - * Defaults to false until #328 step 3 switches the mobile default. - * - * Separate from `pythonBindings.pythonFreeThreaded` on purpose: that one follows the desktop - * flavour the library was built with, while the library's own iOS klib is still compiled against - * the GIL headers. Deriving iOS from it would pair a GIL klib with a free-threaded framework the - * moment the library is built with `-PpythonFreeThreaded=true`. Agent proposal, not confirmed. - */ - val freeThreaded: Property } /** @@ -73,14 +61,15 @@ internal object FreeThreadedMobileRelease { /** One iOS support archive the consumer wiring can download: where from, under which name, which pin. */ internal data class IosArchive(val url: String, val name: String, val sha256: String) -/** The archive [configureIosConsumer] downloads for the selected flavour of [DEFAULT_PYTHON_VERSION]. */ -internal fun iosArchiveFor(freeThreaded: Boolean): IosArchive = - if (freeThreaded) { - val name = FreeThreadedMobileRelease.iosArchiveName(DEFAULT_PYTHON_VERSION) - IosArchive(FreeThreadedMobileRelease.url(DEFAULT_PYTHON_VERSION, name), name, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) - } else { - IosArchive(DEFAULT_IOS_ARCHIVE_URL, DEFAULT_IOS_ARCHIVE_NAME, DEFAULT_IOS_ARCHIVE_SHA256) - } +/** + * The archive [configureIosConsumer] downloads: the project's own free-threaded `3.15t` build of + * [DEFAULT_PYTHON_VERSION]. It is the only iOS build (owner decision #158, SPEC T-2a); there is no + * GIL archive and no switch. + */ +internal fun iosArchive(): IosArchive { + val name = FreeThreadedMobileRelease.iosArchiveName(DEFAULT_PYTHON_VERSION) + return IosArchive(FreeThreadedMobileRelease.url(DEFAULT_PYTHON_VERSION, name), name, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256) +} internal const val DOWNLOAD_IOS_TASK = "downloadPython_ios" internal const val PREPARE_XCFRAMEWORK_TASK = "preparePythonXcframework" @@ -156,7 +145,7 @@ abstract class DownloadPythonIosTask : DefaultTask() { if (expected.isBlank()) { throw GradleException( "There is no pinned SHA-256 for ${archiveName.get()}: the root python-checksums.properties had no " + - "entry for it (free-threaded: '$DEFAULT_IOS_FREE_THREADED_LOCK_KEY') when this plugin was built. " + + "entry for it ('$DEFAULT_IOS_FREE_THREADED_LOCK_KEY') when this plugin was built. " + "Nothing was downloaded.", ) } @@ -256,15 +245,14 @@ internal fun PythonBindingsPlugin.configureIosConsumer(project: Project, extensi if (iosTargets.isEmpty()) return val build = project.layout.buildDirectory - val freeThreaded = extension.ios.freeThreaded.orElse(false) - val archive = freeThreaded.map { iosArchiveFor(it) } + val archive = iosArchive() val download = project.tasks.register(DOWNLOAD_IOS_TASK, DownloadPythonIosTask::class.java) { group = "python" - description = "Downloads and verifies the pinned CPython iOS support archive into build/python-standalone (issue #90)" - archiveUrl.convention(archive.map { it.url }) - archiveName.convention(archive.map { it.name }) - expectedSha256.convention(archive.map { it.sha256 }) + description = "Downloads and verifies the pinned free-threaded CPython 3.15t iOS support archive into build/python-standalone (issue #90)" + archiveUrl.convention(archive.url) + archiveName.convention(archive.name) + expectedSha256.convention(archive.sha256) workDirectory.convention(build.dir("python-standalone")) } val xcframeworkSource = download.flatMap { task -> task.workDirectory.dir("extracted/ios/Python.xcframework") } @@ -301,7 +289,6 @@ internal fun PythonBindingsPlugin.configureIosConsumer(project: Project, extensi xcframework.set(xcframeworkSource) sliceName.set(slice.name) pythonVersion.set(DEFAULT_PYTHON_VERSION) - this.freeThreaded.set(freeThreaded) destinationDir.set(homeRoot.map { it.dir(slice.name) }) } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt index 43d946465..572e0a069 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosPythonHome.kt @@ -23,8 +23,8 @@ import java.nio.file.StandardCopyOption * * ``` * Python.xcframework/ - * lib/python3.14/ pure-Python stdlib, shared by every slice - * ios-arm64/lib-arm64/python3.14/ lib-dynload/ (the .so files), _sysconfigdata (device) + * lib/python3.15t/ pure-Python stdlib, shared by every slice + * ios-arm64/lib-arm64/python3.15t/ lib-dynload/ (the .so files), _sysconfigdata (device) * ios-arm64_x86_64-simulator/lib-arm64/... the same for the arm64 simulator * ios-arm64_x86_64-simulator/lib-x86_64/... ...and the x86_64 simulator * build/iOS-dylib-Info-template.plist Info.plist for an extension-module framework @@ -117,27 +117,27 @@ object IosPythonHomeLayout { } /** - * `3.14` from `3.14.7`; `3.15t` from `3.15.0` for a free-threaded framework (SPEC T-2a), whose - * standard library is `lib/python3.15t/` and whose slices carry `lib-/python3.15t/`. + * `3.15t` from `3.15.0`: the iOS framework is always free-threaded (SPEC T-2a), so its standard + * library is `lib/python3.15t/` and its slices carry `lib-/python3.15t/`. */ - fun stdlibTag(pythonVersion: String, freeThreaded: Boolean = false): String { + fun stdlibTag(pythonVersion: String): String { val parts = pythonVersion.split('.') require(parts.size >= 2) { "python version '$pythonVersion' has no major.minor" } - return "${parts[0]}.${parts[1]}" + if (freeThreaded) "t" else "" + return "${parts[0]}.${parts[1]}t" } /** - * The trees merged into `lib/python/` for [slice], relative to the xcframework, shared tree + * The trees merged into `lib/pythont/` for [slice], relative to the xcframework, shared tree * first: the slice tree only adds files (`lib-dynload/`, `_sysconfigdata__ios_*`), but if it ever * overlapped, the slice-specific file is the one that must win, as in `install_stdlib`. */ - fun sourceDirectories(slice: Slice, pythonVersion: String, freeThreaded: Boolean = false): List { - val tag = stdlibTag(pythonVersion, freeThreaded) + fun sourceDirectories(slice: Slice, pythonVersion: String): List { + val tag = stdlibTag(pythonVersion) return listOf("lib/python$tag", "${slice.xcframeworkSlice}/lib-${slice.arch}/python$tag") } /** - * Whether a file or directory at [stdlibRelativePath] -- relative to `lib/python/`, `/` + * Whether a file or directory at [stdlibRelativePath] -- relative to `lib/pythont/`, `/` * separated -- goes into the app. * * Not copied: any `__pycache__` (bytecode from whatever ran against the extracted tree, so the @@ -161,10 +161,10 @@ object IosPythonHomeLayout { * ([sourceDirectories]' order). Fails when a source tree is missing, rather than staging half a * prefix that would abort `Py_Initialize()` on the device. */ - fun plan(xcframework: File, slice: Slice, pythonVersion: String, freeThreaded: Boolean = false): Map { - val tag = stdlibTag(pythonVersion, freeThreaded) + fun plan(xcframework: File, slice: Slice, pythonVersion: String): Map { + val tag = stdlibTag(pythonVersion) val result = sortedMapOf() - for (relative in sourceDirectories(slice, pythonVersion, freeThreaded)) { + for (relative in sourceDirectories(slice, pythonVersion)) { val root = File(xcframework, relative) if (!root.isDirectory) { throw GradleException( @@ -186,7 +186,7 @@ object IosPythonHomeLayout { } /** - * Stages one slice's prefix for an iOS app bundle (SPEC L-11): `/home/lib/python/` + * Stages one slice's prefix for an iOS app bundle (SPEC L-11): `/home/lib/pythont/` * as [IosPythonHomeLayout.plan] lists it, plus `/dylib-Info-template.plist` *beside* * the prefix -- it is an input to the Xcode phase, not part of what the app carries. * @@ -207,14 +207,6 @@ abstract class StageIosPythonHomeTask : DefaultTask() { @get:Input abstract val pythonVersion: Property - /** - * Whether [xcframework] is a free-threaded build (`lib/pythont/`, SPEC T-2a). Unset means - * false, the GIL layout. - */ - @get:Input - @get:org.gradle.api.tasks.Optional - abstract val freeThreaded: Property - @get:OutputDirectory abstract val destinationDir: DirectoryProperty @@ -224,8 +216,7 @@ abstract class StageIosPythonHomeTask : DefaultTask() { ?: throw GradleException("unknown iOS slice '${sliceName.get()}'") val framework = xcframework.get().asFile val version = pythonVersion.get() - val ft = freeThreaded.getOrElse(false) - val plan = IosPythonHomeLayout.plan(framework, slice, version, ft) + val plan = IosPythonHomeLayout.plan(framework, slice, version) val root = destinationDir.get().asFile val home = File(root, PREFIX_DIRECTORY) @@ -245,7 +236,7 @@ abstract class StageIosPythonHomeTask : DefaultTask() { Files.copy(template.toPath(), File(root, TEMPLATE_NAME).toPath(), StandardCopyOption.REPLACE_EXISTING) // The marker PythonHomeCheck probes on the device, checked here where it is cheap to fail. - val marker = File(home, "lib/python${IosPythonHomeLayout.stdlibTag(version, ft)}/os.py") + val marker = File(home, "lib/python${IosPythonHomeLayout.stdlibTag(version)}/os.py") if (!marker.isFile) { throw GradleException("staged ${slice.name} into $home but $marker is missing; the app would abort at Py_Initialize().") } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt index 141c59bff..3d7de7808 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosConsumerWiringTest.kt @@ -11,7 +11,6 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse -import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue @@ -147,8 +146,8 @@ class IosConsumerWiringTest { val build = project.layout.buildDirectory.get().asFile assertEquals(File(build, "python-standalone"), task.workDirectory.get().asFile) assertTrue(task.extractDirectory.startsWith(File(build, "python-standalone"))) - assertEquals(DEFAULT_IOS_ARCHIVE_SHA256, task.expectedSha256.get()) - assertEquals(64, DEFAULT_IOS_ARCHIVE_SHA256.length) + assertEquals(DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256, task.expectedSha256.get()) + assertEquals(64, DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256.length) } @Test @@ -160,27 +159,12 @@ class IosConsumerWiringTest { assertEquals(File(build, "python-ios-home/${IosPythonHomeLayout.slices[0].name}"), stage.destinationDir.get().asFile) } - // ---- free-threaded archive (SPEC T-2a, #328 step 1) --------------------------------------- + // ---- the 3.15t archive is the only one (SPEC T-2a, owner decision #158) -------------------- @Test - fun theDefaultIsStillTheGilArchiveAndLayout() { - // Guard, not a red test: step 1 must not switch the mobile default (#328 step 3 does). + fun theDefaultIsTheProjectsOwn315tArchiveAndLayout() { val project = project() wire(project) - val task = project.tasks.getByName(DOWNLOAD_IOS_TASK) as DownloadPythonIosTask - assertEquals(DEFAULT_IOS_ARCHIVE_URL, task.archiveUrl.get()) - assertEquals(DEFAULT_IOS_ARCHIVE_NAME, task.archiveName.get()) - assertEquals(DEFAULT_IOS_ARCHIVE_SHA256, task.expectedSha256.get()) - for (slice in IosPythonHomeLayout.slices) { - val stage = project.tasks.getByName(slice.stageTaskName) as StageIosPythonHomeTask - assertEquals(false, stage.freeThreaded.get(), slice.name) - } - } - - @Test - fun freeThreadedSelectsTheProjectsOwnReleaseArchive() { - val project = project() - wire(project) { ios { freeThreaded.set(true) } } val v = DEFAULT_PYTHON_VERSION val task = project.tasks.getByName(DOWNLOAD_IOS_TASK) as DownloadPythonIosTask assertEquals( @@ -191,8 +175,9 @@ class IosConsumerWiringTest { assertEquals(DEFAULT_IOS_FREE_THREADED_ARCHIVE_SHA256, task.expectedSha256.get()) for (slice in IosPythonHomeLayout.slices) { val stage = project.tasks.getByName(slice.stageTaskName) as StageIosPythonHomeTask - assertEquals(true, stage.freeThreaded.get(), slice.name) + assertEquals(v, stage.pythonVersion.get(), slice.name) } + assertTrue(IosPythonHomeLayout.stdlibTag(v).endsWith("t"), "the staged stdlib is lib/pythont/") } @Test @@ -280,7 +265,7 @@ class IosConsumerWiringTest { @Test fun aMissingPinFailsBeforeAnyDownload() { - // A plugin built from a python-checksums.properties without the free-threaded key bakes "". + // A plugin built from a python-checksums.properties without the iOS key bakes "". val bytes = tarGz(mapOf("Python.xcframework/Info.plist" to "plist")) val (task, _) = downloadTask(bytes, "") val e = assertFailsWith { task.provision() } @@ -316,6 +301,5 @@ class IosConsumerWiringTest { assertEquals("ios-arm64_x86_64-simulator", xcframeworkSliceForKonanTarget("ios_simulator_arm64")) assertEquals("ios-arm64_x86_64-simulator", xcframeworkSliceForKonanTarget("ios_x64")) assertNull(xcframeworkSliceForKonanTarget("linux_x64")) - assertNotNull(DEFAULT_IOS_ARCHIVE_URL) } } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt index 5c3d01df6..a96b4a111 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/IosPythonHomeLayoutTest.kt @@ -19,12 +19,12 @@ import kotlin.test.assertTrue * The two share no code, so each pins the directory name as a literal. * * What this cannot find: whether CPython on a device really loads the result. The fake tree below - * mirrors the observed 3.14 archive's *shape*; the installed-app run in + * mirrors the observed 3.15.0t archive's *shape*; the installed-app run in * `docs/platforms/ios-app-bundle.md` is the check for the rest. */ class IosPythonHomeLayoutTest { - private val version = "3.14.7" + private val version = "3.15.0" // Under this module's build directory (the test task's working directory), not the system // temporary directory: AGENTS.md rule 2. private val scratch: File = @@ -94,7 +94,7 @@ class IosPythonHomeLayoutTest { fun theSharedStdlibComesFirstAndTheSliceTreeSecond() { val sim = IosPythonHomeLayout.sliceFor("-iphonesimulator", "x86_64") assertEquals( - listOf("lib/python3.14", "ios-arm64_x86_64-simulator/lib-x86_64/python3.14"), + listOf("lib/python3.15t", "ios-arm64_x86_64-simulator/lib-x86_64/python3.15t"), IosPythonHomeLayout.sourceDirectories(sim, version), ) } @@ -105,8 +105,8 @@ class IosPythonHomeLayoutTest { assertTrue(staged("os.py")) assertTrue(staged("encodings/__init__.py")) assertTrue(staged("json/decoder.py")) - assertTrue(staged("lib-dynload/_json.cpython-314-iphonesimulator.so")) - assertTrue(staged("_sysconfigdata__ios_arm64-iphonesimulator.py")) + assertTrue(staged("lib-dynload/_json.cpython-315t-iphonesimulator.so")) + assertTrue(staged("_sysconfigdata_t_ios_arm64-iphonesimulator.py")) assertTrue(staged("site-packages/README.txt")) // A package *named like* an excluded one deeper down is not excluded. assertTrue(staged("email/test/__init__.py")) @@ -117,9 +117,9 @@ class IosPythonHomeLayoutTest { assertFalse(staged("idlelib/idle.py")) assertFalse(staged("turtledemo/__main__.py")) assertFalse(staged("ensurepip/_bundled/pip.whl")) - assertFalse(staged("encodings/__pycache__/aliases.cpython-314.pyc")) + assertFalse(staged("encodings/__pycache__/aliases.cpython-315t.pyc")) assertFalse(staged("__pycache__")) - assertFalse(staged("libpython3.14.dylib")) + assertFalse(staged("libpython3.15t.dylib")) } @Test @@ -129,10 +129,10 @@ class IosPythonHomeLayoutTest { val plan = IosPythonHomeLayout.plan(xcframework, slice, version) assertEquals( listOf( - "lib/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py", - "lib/python3.14/encodings/__init__.py", - "lib/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so", - "lib/python3.14/os.py", + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", + "lib/python3.15t/os.py", ), plan.keys.toList(), ) @@ -160,7 +160,7 @@ class IosPythonHomeLayoutTest { .build() val out = File(scratch, "out") // Left over from a previous run with a wider selection; must not survive. - File(out, "home/lib/python3.14/test/stale.py").apply { parentFile.mkdirs(); writeText("") } + File(out, "home/lib/python3.15t/test/stale.py").apply { parentFile.mkdirs(); writeText("") } val task = project.tasks.create("stage", StageIosPythonHomeTask::class.java) task.xcframework.set(xcframework) @@ -173,10 +173,10 @@ class IosPythonHomeLayoutTest { .map { it.relativeTo(File(out, "home")).invariantSeparatorsPath }.sorted().toList() assertEquals( listOf( - "lib/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py", - "lib/python3.14/encodings/__init__.py", - "lib/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so", - "lib/python3.14/os.py", + "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", + "lib/python3.15t/encodings/__init__.py", + "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", + "lib/python3.15t/os.py", ), staged, ) @@ -184,113 +184,42 @@ class IosPythonHomeLayoutTest { assertFalse(File(out, "home/${StageIosPythonHomeTask.TEMPLATE_NAME}").exists()) } - // ---- free-threaded (SPEC T-2a, #328) ---- - - @Test - fun freeThreadedSourceDirectoriesCarryTheTSuffix() { - val sim = IosPythonHomeLayout.sliceFor("-iphonesimulator", "x86_64") - assertEquals( - listOf("lib/python3.15t", "ios-arm64_x86_64-simulator/lib-x86_64/python3.15t"), - IosPythonHomeLayout.sourceDirectories(sim, "3.15.0", freeThreaded = true), - ) - assertEquals("3.15t", IosPythonHomeLayout.stdlibTag("3.15.0", freeThreaded = true)) - assertEquals("3.15", IosPythonHomeLayout.stdlibTag("3.15.0")) - } + // ---- the layout is always free-threaded (SPEC T-2a, owner decision #158) ---- @Test - fun aFreeThreadedPlanStagesLibPython315t() { - val xcframework = fakeFreeThreadedXcframework() - val slice = IosPythonHomeLayout.sliceFor("-iphonesimulator", "arm64") - val plan = IosPythonHomeLayout.plan(xcframework, slice, "3.15.0", freeThreaded = true) - assertEquals( - listOf( - "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py", - "lib/python3.15t/encodings/__init__.py", - "lib/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so", - "lib/python3.15t/os.py", - ), - plan.keys.toList(), - ) - assertFalse(plan.values.any { it.name.startsWith("libpython") }, "no libpython*.dylib: ${plan.values}") + fun theStdlibTagAlwaysCarriesTheTSuffix() { + assertEquals("3.15t", IosPythonHomeLayout.stdlibTag("3.15.0")) + assertEquals("3.15t", IosPythonHomeLayout.stdlibTag(version)) } @Test - fun aGilPlanAgainstAFreeThreadedTreeFails() { - // Guard, not a red test: the GIL tag must not silently find a free-threaded tree. - val xcframework = fakeFreeThreadedXcframework() + fun aGilShapedTreeIsNotFoundByThePlan() { + // Guard: a framework with only lib/python3.15 (no t suffix) must fail loudly, not stage. + val root = File(scratch, "gil/Python.xcframework") + fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } + file("lib/python3.15/os.py") + file("ios-arm64/lib-arm64/python3.15/lib-dynload/_json.cpython-315-iphoneos.so") assertFailsWith { - IosPythonHomeLayout.plan(xcframework, IosPythonHomeLayout.sliceFor("-iphoneos", "arm64"), "3.15.0") + IosPythonHomeLayout.plan(root, IosPythonHomeLayout.sliceFor("-iphoneos", "arm64"), version) } } - @Test - fun theTaskStagesAFreeThreadedPrefix() { - val xcframework = fakeFreeThreadedXcframework() - val project = ProjectBuilder.builder() - .withProjectDir(File(scratch, "project").apply { mkdirs() }) - .withGradleUserHomeDir(File(scratch, "gradle-home")) - .build() - val out = File(scratch, "out-ft") - val task = project.tasks.create("stageFt", StageIosPythonHomeTask::class.java) - task.xcframework.set(xcframework) - task.sliceName.set("iphoneos-arm64") - task.pythonVersion.set("3.15.0") - task.freeThreaded.set(true) - task.destinationDir.set(out) - task.stage() - - val staged = File(out, "home").walkTopDown().filter { it.isFile } - .map { it.relativeTo(File(out, "home")).invariantSeparatorsPath }.sorted().toList() - assertEquals( - listOf( - "lib/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py", - "lib/python3.15t/encodings/__init__.py", - "lib/python3.15t/lib-dynload/_json.cpython-315t-iphoneos.so", - "lib/python3.15t/os.py", - ), - staged, - ) - } - - /** - * The shape of `python-3.15.0t-iOS-XCframework.tar.gz` (release `cpython-3.15.0t-mobile`), taken - * from `tar tzf` of the published archive: `lib/python3.15t` shared, `lib-/python3.15t` per - * slice, `libpython3.15t.dylib` beside them. - */ - private fun fakeFreeThreadedXcframework(): File { - val root = File(scratch, "ft/Python.xcframework") + /** The observed 3.15.0t archive's shape, a few files deep. */ + private fun fakeXcframework(): File { + val root = File(scratch, "Python.xcframework") fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } file("lib/python3.15t/os.py") file("lib/python3.15t/encodings/__init__.py") - file("lib/python3.15t/test/test_os/test_os.py") - file("ios-arm64/lib/libpython3.15t.dylib") - file("ios-arm64/lib-arm64/libpython3.15t.dylib") - file("ios-arm64/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py") + file("lib/python3.15t/encodings/__pycache__/__init__.cpython-315t.pyc") + file("lib/python3.15t/test/test_os.py") + file("lib/python3.15t/tkinter/__init__.py") file("ios-arm64/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphoneos.so") - file("ios-arm64_x86_64-simulator/lib/libpython3.15t.dylib") - file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py") + file("ios-arm64/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphoneos.py") + file("ios-arm64/lib/libpython3.15t.dylib") file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") + file("ios-arm64_x86_64-simulator/lib-arm64/python3.15t/_sysconfigdata_t_ios_arm64-iphonesimulator.py") file("ios-arm64_x86_64-simulator/lib-x86_64/python3.15t/lib-dynload/_json.cpython-315t-iphonesimulator.so") - file("build/iOS-dylib-Info-template.plist") - return root - } - - /** The observed 3.14 archive's shape, a few files deep. */ - private fun fakeXcframework(): File { - val root = File(scratch, "Python.xcframework") - fun file(path: String) = File(root, path).apply { parentFile.mkdirs(); writeText(path) } - file("lib/python3.14/os.py") - file("lib/python3.14/encodings/__init__.py") - file("lib/python3.14/encodings/__pycache__/__init__.cpython-314.pyc") - file("lib/python3.14/test/test_os.py") - file("lib/python3.14/tkinter/__init__.py") - file("ios-arm64/lib-arm64/python3.14/lib-dynload/_json.cpython-314-iphoneos.so") - file("ios-arm64/lib-arm64/python3.14/_sysconfigdata__ios_arm64-iphoneos.py") - file("ios-arm64/lib/libpython3.14.dylib") - file("ios-arm64_x86_64-simulator/lib-arm64/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so") - file("ios-arm64_x86_64-simulator/lib-arm64/python3.14/_sysconfigdata__ios_arm64-iphonesimulator.py") - file("ios-arm64_x86_64-simulator/lib-x86_64/python3.14/lib-dynload/_json.cpython-314-iphonesimulator.so") - file("ios-arm64_x86_64-simulator/lib/libpython3.14.dylib") + file("ios-arm64_x86_64-simulator/lib/libpython3.15t.dylib") file("build/iOS-dylib-Info-template.plist") return root } From bffb9bfa50fa1a0d7c7eae0f04491a6b66322a1f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:27:44 +0900 Subject: [PATCH 39/61] build-ft-mobile: build free-threaded CPython for x86_64 Android too (#328) Owner approval 2026-11-13: keep arm64-v8a and x86_64 both supported on 3.15t. build-android.sh takes the host as a second argument and checks the ELF architecture per host; the Android job is a matrix. --- .github/scripts/ft-mobile/build-android.sh | 14 ++++++++++---- .github/workflows/build-ft-mobile.yml | 13 +++++++++---- docs/platforms/free-threaded-mobile.md | 6 +++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/scripts/ft-mobile/build-android.sh b/.github/scripts/ft-mobile/build-android.sh index cc5ced80d..cb9500cf8 100755 --- a/.github/scripts/ft-mobile/build-android.sh +++ b/.github/scripts/ft-mobile/build-android.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Build free-threaded CPython (--disable-gil) for aarch64-linux-android with -# CPython's own tooling (Platforms/Android). Usage: build-android.sh [VERSION] +# CPython's own tooling (Platforms/Android). Usage: build-android.sh [VERSION] [HOST] +# HOST is aarch64-linux-android (default) or x86_64-linux-android. # Needs ANDROID_HOME (SDK with cmdline-tools/latest) and a JDK 17, 21 or 25. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" @@ -8,7 +9,12 @@ here="$(cd "$(dirname "$0")" && pwd)" source "$here/common.sh" PY_VERSION="${1:-3.15.0}" -HOST=aarch64-linux-android +HOST="${2:-aarch64-linux-android}" +case "$HOST" in + aarch64-linux-android) elf_arch="ARM aarch64" ;; + x86_64-linux-android) elf_arch="x86-64" ;; + *) echo "unsupported HOST: $HOST" >&2; exit 2 ;; +esac : "${ANDROID_HOME:?ANDROID_HOME must point at the Android SDK}" src="$(fetch_source "$PY_VERSION")" @@ -55,9 +61,9 @@ require_gil_disabled "$check/prefix/include/python${minor}t/pyconfig.h" lib="$(ls "$check"/prefix/lib/libpython${minor}t*.so* 2>/dev/null | head -n 1 || true)" [ -n "$lib" ] || die "no libpython${minor}t*.so in the package" log "ok: $lib" -file "$lib" | tee /dev/stderr | grep -q 'ARM aarch64' || die "$lib is not an aarch64 ELF" +file "$lib" | tee /dev/stderr | grep -q "$elf_arch" || die "$lib is not an $elf_arch ELF" [ -d "$check/prefix/lib/python${minor}t" ] || die "no lib/python${minor}t in the package" cp "$archive" "$FT_OUT/" -echo "Android archive:" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +echo "Android archive ($HOST):" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" record_sha256 "$FT_OUT/$(basename "$archive")" diff --git a/.github/workflows/build-ft-mobile.yml b/.github/workflows/build-ft-mobile.yml index 0b9d05954..1cc250d0d 100644 --- a/.github/workflows/build-ft-mobile.yml +++ b/.github/workflows/build-ft-mobile.yml @@ -1,6 +1,6 @@ name: Build free-threaded CPython for mobile -# Builds CPython with --disable-gil ("3.15t") for Android arm64 and iOS with +# Builds CPython with --disable-gil ("3.15t") for Android (aarch64 and x86_64) and iOS with # CPython's own tooling (Platforms/Android, Platforms/Apple). Manual only: the # builds take a long time and the archives are inputs to later work (#163). # See docs/platforms/free-threaded-mobile.md. @@ -27,7 +27,12 @@ concurrency: jobs: android: - name: Android aarch64 (free-threaded) + # Both ABIs the library packages (arm64-v8a, x86_64); x86_64 approved by the owner 2026-11-13 (#328). + name: Android ${{ matrix.host }} (free-threaded) + strategy: + fail-fast: false + matrix: + host: [aarch64-linux-android, x86_64-linux-android] runs-on: ubuntu-24.04 timeout-minutes: 120 env: @@ -48,11 +53,11 @@ jobs: libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev - name: Build - run: .github/scripts/ft-mobile/build-android.sh "${{ inputs.python_version || '3.15.0' }}" + run: .github/scripts/ft-mobile/build-android.sh "${{ inputs.python_version || '3.15.0' }}" "${{ matrix.host }}" - uses: actions/upload-artifact@v4 with: - name: python-${{ inputs.python_version || '3.15.0' }}t-android-aarch64 + name: python-${{ inputs.python_version || '3.15.0' }}t-${{ matrix.host }} path: ${{ env.FT_OUT }}/*.tar.gz if-no-files-found: error diff --git a/docs/platforms/free-threaded-mobile.md b/docs/platforms/free-threaded-mobile.md index 8146231ae..598aeddf8 100644 --- a/docs/platforms/free-threaded-mobile.md +++ b/docs/platforms/free-threaded-mobile.md @@ -12,7 +12,7 @@ CPython's own mobile tooling. | Platform | Host | Tooling | Output | |---|---|---|---| -| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0-aarch64-linux-android.tar.gz` | +| Android | `aarch64-linux-android`, `x86_64-linux-android` (owner approval 2026-11-13, #328: both ABIs the library packages) | `Platforms/Android` | `python-3.15.0-.tar.gz`, one per host | | iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0-iOS-XCframework.tar.gz` | The iOS tooling only builds an XCframework from all of its slices, so the simulator @@ -96,13 +96,13 @@ Each script fails the job unless: - the build Python reports `Py_GIL_DISABLED == 1`; - Android: `include/python3.15t/pyconfig.h` has `#define Py_GIL_DISABLED 1`, - `lib/libpython3.15t*.so` exists and is an aarch64 ELF, `lib/python3.15t` exists; + `lib/libpython3.15t*.so` exists and is an ELF of the host's architecture (aarch64 or x86-64), `lib/python3.15t` exists; - iOS: every slice has `Headers/pyconfig-.h` with `#define Py_GIL_DISABLED 1`, `lib/libpython3.15t.dylib`, and a `Python.framework/Python` binary (`lipo -info` is logged); the XCframework has `lib/python3.15t` and exactly two slices. The sha256 of each archive is written to the job summary and the archives are uploaded -as workflow artifacts (`python-t-android-aarch64`, +as workflow artifacts (`python-t-aarch64-linux-android`, `python-t-x86_64-linux-android`, `python-t-ios-xcframework`). ## Not verified From 58ba0114fcd1f60c07a29bd0fb7e29389ccf65d6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:36:13 +0900 Subject: [PATCH 40/61] CI: build free-threaded CPython 3.15t for wasm32-emscripten with pthreads (#175) Owner approval 2026-11-13. New build-ft-wasm.yml (dispatch, push on develop and pull requests on its own paths) and .github/scripts/ft-wasm/build-wasm.sh: --disable-gil --enable-wasm-pthreads, libffi and mpdec with -pthread, a call-trampoline patch for shared memory, then a node smoke test that starts four threads with the GIL disabled. Artifact only; no release. docs/platforms/free-threaded-wasm.md. --- .github/scripts/ft-wasm/build-wasm.sh | 249 ++++++++++++++++++++++++++ .github/scripts/ft-wasm/smoke.py | 42 +++++ .github/workflows/build-ft-wasm.yml | 85 +++++++++ docs/platforms/free-threaded-wasm.md | 153 ++++++++++++++++ docs/platforms/wasm-design.md | 4 + 5 files changed, 533 insertions(+) create mode 100755 .github/scripts/ft-wasm/build-wasm.sh create mode 100644 .github/scripts/ft-wasm/smoke.py create mode 100644 .github/workflows/build-ft-wasm.yml create mode 100644 docs/platforms/free-threaded-wasm.md diff --git a/.github/scripts/ft-wasm/build-wasm.sh b/.github/scripts/ft-wasm/build-wasm.sh new file mode 100755 index 000000000..328861309 --- /dev/null +++ b/.github/scripts/ft-wasm/build-wasm.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Build free-threaded CPython (--disable-gil) with pthreads for wasm32-emscripten, +# with CPython's own driver (Platforms/emscripten). Step one of #175: a build +# experiment. Usage: build-wasm.sh [VERSION] +# +# Environment: +# FT_WORK scratch directory (default .tmp/ft-wasm) +# FT_OUT receives the archive the workflow uploads (default .tmp/ft-wasm-out) +# FT_WASM_EMSDK_VERSION Emscripten to install; empty = the version CPython itself pins +# (Platforms/emscripten/config.toml, 4.0.19 in 3.15.0) +# FT_WASM_TRAMPOLINE_PATCH apply (default) | skip; see patch_trampoline below +# +# Needs: a C toolchain for the native build Python, git, curl, node on PATH +# (CPython 3.15.0 pins node 24 in config.toml), python3 >= 3.11 to run the driver. +# +# This is a separate script, not a mode of python-multiplatform/scripts/wasm/build-cpython.sh: +# that script is pinned to CPython 3.14.2 / Emscripten 5.0.3 / pyemscripten_2026_0, clones a +# pinned git commit, writes into .caches/ and stages the runtime the wasmJs suite runs against. +# This job builds a different CPython (3.15.0 tarball) with a different Emscripten and must not +# touch that runtime. The flags below are the #175 investigation's proposal, not a decision. +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" + +export FT_WORK="${FT_WORK:-$PWD/.tmp/ft-wasm}" +export FT_OUT="${FT_OUT:-$PWD/.tmp/ft-wasm-out}" +# fetch_source, require_gil_disabled, record_sha256, py_minor +# shellcheck source-path=SCRIPTDIR source=../ft-mobile/common.sh +source "$here/../ft-mobile/common.sh" +log() { echo "[ft-wasm] $*" >&2; } +die() { echo "[ft-wasm] ERROR: $*" >&2; exit 1; } + +PY_VERSION="${1:-3.15.0}" +TRAMPOLINE_PATCH="${FT_WASM_TRAMPOLINE_PATCH:-apply}" +minor="$(py_minor "$PY_VERSION")" +HOST=wasm32-emscripten + +# Compile and link flags (agent proposal from the #175 investigation, not confirmed). +# The driver hardcodes CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 and appends user args after +# it; autoconf keeps the last assignment, so the full string is repeated here. +# -fwasm-exceptions -sSUPPORT_LONGJMP=wasm same unwinding ABI as build-cpython.sh +# -pthread atomics + shared memory (configure also adds it +# to CFLAGS_NODIST for --enable-wasm-pthreads) +# -sPTHREAD_POOL_SIZE=8 pre-created Workers: Thread.start() blocks until +# the thread runs, and a Worker cannot start while +# the JS thread that asked for it is blocked +# -sMAXIMUM_MEMORY=2GB shared memory needs a declared maximum +ABI_FLAGS="-fwasm-exceptions -sSUPPORT_LONGJMP=wasm" +HOST_CFLAGS="-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 $ABI_FLAGS -pthread" +HOST_LDFLAGS="$ABI_FLAGS -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" + +src="$(fetch_source "$PY_VERSION")" +cd "$src" +[ -d Platforms/emscripten ] || die "no Platforms/emscripten in CPython $PY_VERSION (3.15+ expected)" + +# ---- Emscripten ----------------------------------------------------------- +# One emsdk under FT_WORK. The driver is then run without --emsdk-cache, so it uses the +# activated environment ("Build will use EMSDK from current environment"). +pinned="$(sed -n 's/^emscripten-version = "\(.*\)"$/\1/p' Platforms/emscripten/config.toml)" +[ -n "$pinned" ] || die "no emscripten-version in Platforms/emscripten/config.toml" +EMSDK_VERSION="${FT_WASM_EMSDK_VERSION:-$pinned}" +log "Emscripten pinned by CPython $PY_VERSION: $pinned; using: $EMSDK_VERSION" +emsdk="$FT_WORK/emsdk" +if [ ! -x "$emsdk/emsdk" ]; then + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$emsdk" +fi +"$emsdk/emsdk" install "$EMSDK_VERSION" +"$emsdk/emsdk" activate "$EMSDK_VERSION" +# shellcheck disable=SC1091 +EMSDK_QUIET=1 source "$emsdk/emsdk_env.sh" +emcc --version > "$FT_WORK/emcc-version.txt" +emcc_line="$(head -n 1 "$FT_WORK/emcc-version.txt")" +log "$emcc_line" +case "$emcc_line" in *" $EMSDK_VERSION "*) ;; *) die "emcc is not $EMSDK_VERSION" ;; esac +node --version + +# ---- patch: call trampoline under shared memory --------------------------- +# Python/emscripten_trampoline.c instantiates a wasm-gc trampoline module in pre-run. That +# module is compiled with -Wl,--import-memory and no -pthread (Makefile.pre.in), so it imports +# a NON-shared env.memory; giving it the shared wasmMemory of a pthread build is a link error, +# and the `new WebAssembly.Instance` call is outside the try block, so startup throws. +# Pyodide #237 also reports that the addFunction'ed pointer exists only in the main thread's +# table ("table index is out of bounds" on the first call from a spawned thread). +# Minimal fix: when memory is shared, return 0, which selects the existing JS trampoline +# (_PyEM_TrampolineCall_JS, wasmTable.get(func) on the calling thread). The edit is asserted, +# so an upstream change fails the build instead of silently skipping the patch. +# FT_WASM_TRAMPOLINE_PATCH=skip leaves the source alone, to observe the unpatched failure. +patch_trampoline() { + python3 - Python/emscripten_trampoline.c <<'PATCH' +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() +marker = "pmp-175: shared memory" +if marker not in t: + old = "function getPyEMTrampolinePtr() {\n" + new = (old + + " // " + marker + ": the wasm-gc trampoline imports non-shared memory and its\n" + " // addFunction pointer is valid only on this thread; use the JS trampoline.\n" + " if (typeof SharedArrayBuffer !== \"undefined\" && wasmMemory.buffer instanceof SharedArrayBuffer) {\n" + " return 0;\n" + " }\n") + assert t.count(old) == 1, "getPyEMTrampolinePtr changed shape; re-check the patch" + p.write_text(t.replace(old, new)) + print("patched Python/emscripten_trampoline.c (shared memory -> JS trampoline)") +PATCH +} +case "$TRAMPOLINE_PATCH" in + apply) patch_trampoline ;; + skip) log "trampoline patch skipped (FT_WASM_TRAMPOLINE_PATCH=skip)" ;; + *) die "FT_WASM_TRAMPOLINE_PATCH must be apply or skip, got '$TRAMPOLINE_PATCH'" ;; +esac + +# ---- native build Python (free-threaded too, as in ft-mobile) ------------- +python3 --version +python3 Platforms/emscripten configure-build-python -- --disable-gil +python3 Platforms/emscripten make-build-python +python3 Platforms/emscripten pythoninfo-build +cross-build/build/python -c 'import sys, sysconfig; assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1; print("build python free-threaded:", sys.version)' + +# ---- libffi and mpdec with -pthread --------------------------------------- +# The driver builds them with CFLAGS="-O2 -fPIC -DWASM_BIGINT" (make_libffi.sh) and +# CFLAGS=-fPIC (mpdec). emcc appends EMCC_CFLAGS to every invocation, so -pthread reaches +# both without editing the driver. Checked below by the objects' target_features section. +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-libffi +EMCC_CFLAGS="-pthread" python3 Platforms/emscripten make-mpdec + +prefix="cross-build/$HOST/prefix" +for lib in "$prefix/lib/libffi.a" "$prefix/lib/libmpdec.a"; do + [ -f "$lib" ] || die "no $lib" + # wasm target_features entries are : "+", 7, "atomics". + python3 - "$lib" <<'CHECK' || die "$lib was not compiled with atomics (-pthread)" +import sys +data = open(sys.argv[1], "rb").read() +sys.exit(0 if b"+\x07atomics" in data else 1) +CHECK + log "ok: $lib has +atomics" +done + +# ---- host configure ------------------------------------------------------- +python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=$HOST_CFLAGS" \ + "LDFLAGS=$HOST_LDFLAGS" \ + --disable-gil --enable-wasm-pthreads + +hostdir="cross-build/$HOST/build/python" +require_gil_disabled "$hostdir/pyconfig.h" +# configure adds -sUSE_PTHREADS to LDFLAGS_NODIST (CONFIGURE_LDFLAGS_NODIST in the +# Makefile) only when --enable-wasm-pthreads took effect. +grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \ + || die "configure did not enable wasm pthreads (no -sUSE_PTHREADS in LDFLAGS_NODIST)" +grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t" +log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t" + +# ---- host build ----------------------------------------------------------- +# make-host also runs `python.sh --version`, so a startup failure (for example the trampoline +# instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step. +python3 Platforms/emscripten make-host + +# ---- verification --------------------------------------------------------- +# python.wasm must use shared memory with a declared maximum. +node - "$hostdir/python.wasm" <<'NODE' || die "python.wasm has no shared memory" +const b = require("fs").readFileSync(process.argv[2]); +const leb = (i) => { let r = 0, s = 0; for (;;) { const x = b[i++]; r += (x & 0x7f) * 2 ** s; s += 7; if (!(x & 0x80)) return [r, i]; } }; +const str = (i) => { let n; [n, i] = leb(i); return [b.slice(i, i + n).toString(), i + n]; }; +const limits = (i) => { const flags = b[i++]; let min, max = null; [min, i] = leb(i); if (flags & 1) [max, i] = leb(i); return [{ flags, min, max }, i]; }; +let i = 8, mem = null; +while (i < b.length && !mem) { + const id = b[i++]; let size; [size, i] = leb(i); const end = i + size; + if (id === 2) { let n, j; [n, j] = leb(i); + for (let k = 0; k < n && !mem; k++) { let m, f; [m, j] = str(j); [f, j] = str(j); const kind = b[j++]; + if (kind === 0) { [, j] = leb(j); } + else if (kind === 1) { j++; [, j] = limits(j); } + else if (kind === 2) { let l; [l, j] = limits(j); mem = { where: `import ${m}.${f}`, ...l }; } + else if (kind === 3) { j += 2; } + else if (kind === 4) { j++; [, j] = leb(j); } + else throw new Error(`unknown import kind ${kind}`); } } + if (id === 5) { let j; [, j] = leb(i); const [l] = limits(j); mem = { where: "defined", ...l }; } + i = end; +} +if (!mem) { console.error("no memory found"); process.exit(1); } +const shared = (mem.flags & 2) !== 0; +console.log(`python.wasm memory: ${mem.where} shared=${shared} min=${mem.min} max=${mem.max} pages`); +process.exit(shared && mem.max !== null ? 0 : 1); +NODE + +# The interpreter itself: GIL off at run time, four real threads. +run_smoke() { + local label="$1"; shift + local out="$FT_WORK/smoke-$label.log" rc + set +e + timeout 600 "$@" "$here/smoke.py" > "$out" 2>&1 + rc=$? + set -e + cat "$out" + [ "$rc" -eq 0 ] || die "smoke test ($label) exited $rc (124 = timed out, a thread start may have deadlocked)" + grep -q '^FT-WASM-SMOKE-OK ' "$out" || die "smoke test ($label) did not print FT-WASM-SMOKE-OK" + log "ok: smoke test ($label)" +} +run_smoke build-tree "$PWD/$hostdir/python.sh" + +# ---- archive -------------------------------------------------------------- +# The driver has no install or package step for Emscripten, so the archive is assembled here. +# The stdlib directory is named by the built interpreter's own sysconfig, so a non-FT build +# (python3.15) fails the name check instead of being renamed. +stdlib_name="$(timeout 300 "$PWD/$hostdir/python.sh" -c 'import os, sysconfig; print(os.path.basename(sysconfig.get_path("stdlib")))' | tail -n 1)" +[ "$stdlib_name" = "python${minor}t" ] || die "interpreter's stdlib dir is '$stdlib_name', expected python${minor}t" +log "ok: interpreter reports lib/$stdlib_name" + +name="python-${PY_VERSION}t-$HOST" +stage="$FT_WORK/stage/$name" +rm -rf "$FT_WORK/stage"; mkdir -p "$stage/lib/$stdlib_name/lib-dynload" "$stage/include/$stdlib_name" +cp "$hostdir/python.mjs" "$hostdir/python.wasm" "$stage/" +cp "$hostdir/node_entry.mjs" "$hostdir/streams.mjs" "$stage/" +# Everything but the test suite; the zip-and-omit selection is left to whoever stages it. +tar -C Lib --exclude='./test' --exclude='__pycache__' -cf - . | tar -C "$stage/lib/$stdlib_name" -xf - +cp "$hostdir/$(cat "$hostdir/pybuilddir.txt")"/_sysconfigdata_*.py "$stage/lib/$stdlib_name/" +touch "$stage/lib/$stdlib_name/lib-dynload/.empty" +cp -R Include/. "$stage/include/$stdlib_name/" +cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/" +{ + echo "CPython $PY_VERSION, --disable-gil --enable-wasm-pthreads, $HOST" + echo "emcc: $emcc_line" + echo "CFLAGS=$HOST_CFLAGS" + echo "LDFLAGS=$HOST_LDFLAGS" + echo "trampoline patch: $TRAMPOLINE_PATCH" + echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)" +} > "$stage/BUILD-INFO.txt" + +archive="$FT_OUT/$name.tar.gz" +mkdir -p "$FT_OUT" +tar -C "$FT_WORK/stage" -czf "$archive" "$name" + +# Re-check the archive itself, unpacked somewhere else, and run it with PYTHONHOME pointing +# at it (node_entry.mjs copies process.env into the Emscripten ENV, and NODEFS mounts /). +check="$FT_WORK/wasm-check" +rm -rf "$check"; mkdir -p "$check" +tar -xzf "$archive" -C "$check" +root="$check/$name" +require_gil_disabled "$root/include/python${minor}t/pyconfig.h" +[ -d "$root/lib/python${minor}t" ] || die "no lib/python${minor}t in the archive" +[ -f "$root/lib/python${minor}t/os.py" ] || die "no lib/python${minor}t/os.py in the archive" +ls "$root/lib/python${minor}t"/_sysconfigdata_*.py > /dev/null || die "no _sysconfigdata in the archive" +nodeflags="" +[ "$(node -p 'process.versions.node.split(".")[0]')" = "24" ] && nodeflags="--experimental-wasm-jspi" +# shellcheck disable=SC2086 +run_smoke archive env PYTHONHOME="$root" node $nodeflags "$root/node_entry.mjs" \ + --this-program="$root/bin/python${minor}t" + +echo "wasm32-emscripten archive (Emscripten $EMSDK_VERSION, trampoline patch: $TRAMPOLINE_PATCH):" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +record_sha256 "$archive" diff --git a/.github/scripts/ft-wasm/smoke.py b/.github/scripts/ft-wasm/smoke.py new file mode 100644 index 000000000..78c3758ce --- /dev/null +++ b/.github/scripts/ft-wasm/smoke.py @@ -0,0 +1,42 @@ +# Smoke test for the free-threaded wasm32-emscripten build (#175), run by build-wasm.sh +# under node. Prints one line starting with FT-WASM-SMOKE-OK, which the script greps for; +# any failed assertion exits non-zero instead. +import sys +import sysconfig +import threading + +assert sys.platform == "emscripten", sys.platform +assert sysconfig.get_config_var("Py_GIL_DISABLED") == 1, "not a free-threaded build" +assert sys._is_gil_enabled() is False, "the GIL is enabled at run time" + +N_THREADS = 4 +N_ITER = 200_000 +results = [None] * N_THREADS + + +def work(i): + total = 0 + for k in range(N_ITER): + total += k * (i + 1) + results[i] = (threading.get_ident(), total) + + +threads = [threading.Thread(target=work, args=(i,)) for i in range(N_THREADS)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running after join: {alive}" + +expected = [(i + 1) * N_ITER * (N_ITER - 1) // 2 for i in range(N_THREADS)] +got = [r[1] if r else None for r in results] +assert got == expected, f"results {got} != {expected}" +idents = {r[0] for r in results} +assert len(idents) == N_THREADS, f"expected {N_THREADS} distinct thread idents, got {idents}" +assert threading.get_ident() not in idents, "a worker ran on the main thread" +# Importing threading must not have re-enabled the GIL. +assert sys._is_gil_enabled() is False, "the GIL was re-enabled during the run" + +print(f"FT-WASM-SMOKE-OK gil={sys._is_gil_enabled()} threads={N_THREADS} " + f"abiflags={sys.abiflags} version={sys.version.split()[0]}") diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml new file mode 100644 index 000000000..b0242b7e9 --- /dev/null +++ b/.github/workflows/build-ft-wasm.yml @@ -0,0 +1,85 @@ +name: Build free-threaded CPython for wasm + +# Builds CPython with --disable-gil and pthreads ("3.15t") for wasm32-emscripten with +# CPython's own driver (Platforms/emscripten), then runs it under node with four threads. +# Step one of #175, a build experiment; job approved by the owner on 2026-11-13 ("#175 may +# add the new wasm 3.15t CI build job"). It does not touch wasm.yml or the runtime the +# wasmJs suite uses. See docs/platforms/free-threaded-wasm.md. +# +# Experiment choices (agent proposal, not decisions): Emscripten defaults to the version +# CPython itself pins (Platforms/emscripten/config.toml, 4.0.19 for 3.15.0), not the 5.0.3 +# of python-multiplatform/scripts/wasm/build-cpython.sh; the compile/link flags are the +# #175 investigation's; the call trampoline is patched to fall back to its JS path under +# shared memory (trampoline_patch=skip runs without it). + +on: + # Runs on develop when the build itself changes (workflow_dispatch needs the file on the + # default branch, main, which develop reaches only through a release). + push: + branches: [develop] + paths: + - ".github/workflows/build-ft-wasm.yml" + - ".github/scripts/ft-wasm/**" + - ".github/scripts/ft-mobile/common.sh" + # And on a pull request that changes it, so a change is built before it merges (a new workflow + # cannot be dispatched until it reaches main). + pull_request: + paths: + - ".github/workflows/build-ft-wasm.yml" + - ".github/scripts/ft-wasm/**" + - ".github/scripts/ft-mobile/common.sh" + workflow_dispatch: + inputs: + python_version: + description: CPython version to build (a python.org source release, 3.15 or later) + default: "3.15.0" + required: true + type: string + emscripten_version: + description: Emscripten to use (empty = the version CPython pins in Platforms/emscripten/config.toml) + default: "" + required: false + type: string + trampoline_patch: + description: Call-trampoline shared-memory patch (skip = observe the unpatched build) + default: apply + required: true + type: choice + options: [apply, skip] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + wasm: + name: wasm32-emscripten pthreads (free-threaded) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + FT_WORK: ${{ github.workspace }}/.tmp/ft-wasm + FT_OUT: ${{ github.workspace }}/.tmp/ft-wasm-out + FT_WASM_EMSDK_VERSION: ${{ inputs.emscripten_version || '' }} + FT_WASM_TRAMPOLINE_PATCH: ${{ inputs.trampoline_patch || 'apply' }} + steps: + - uses: actions/checkout@v4 + + # CPython 3.15.0 pins node 24 (Platforms/emscripten/config.toml, node-version). + - uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install build Python prerequisites + run: > + sudo apt-get update && sudo apt-get install -y + build-essential pkg-config libssl-dev zlib1g-dev libffi-dev + libbz2-dev liblzma-dev libsqlite3-dev libzstd-dev + + - name: Build and verify + run: .github/scripts/ft-wasm/build-wasm.sh "${{ inputs.python_version || '3.15.0' }}" + + - uses: actions/upload-artifact@v4 + with: + name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten + path: ${{ env.FT_OUT }}/*.tar.gz + if-no-files-found: error diff --git a/docs/platforms/free-threaded-wasm.md b/docs/platforms/free-threaded-wasm.md new file mode 100644 index 000000000..aab98375c --- /dev/null +++ b/docs/platforms/free-threaded-wasm.md @@ -0,0 +1,153 @@ +# Free-threaded CPython for wasm32-emscripten + +Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI.** Nothing +here has been compiled, linked or run. Tracking issue: #175 (this is its first step, the +build experiment). The owner approved the CI job on 2026-11-13 ("#175 may add the new wasm +3.15t CI build job"). + +python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in +`AGENTS.md`). The wasmJs target today runs a GIL, no-threads CPython 3.14.2 +([`wasm-design.md`](wasm-design.md)). No prebuilt free-threaded CPython with pthreads exists +for Emscripten, so the workflow `.github/workflows/build-ft-wasm.yml` builds one with +CPython's own driver and checks that it runs threads without the GIL under node. It does not +change `wasm.yml`, `python-multiplatform/scripts/wasm/build-cpython.sh` or the runtime the +wasmJs suite uses. + +## What is built + +| Host | Tooling | Output | +|---|---|---| +| `wasm32-emscripten`, pthreads, `--disable-gil` | `Platforms/emscripten` (CPython 3.15) | `python-3.15.0t-wasm32-emscripten.tar.gz` | + +The archive holds `python.mjs`, `python.wasm`, CPython's `node_entry.mjs` and `streams.mjs`, +`lib/python3.15t/` (the standard library without `test/`, plus `_sysconfigdata_*.py` and an +empty `lib-dynload/`), `include/python3.15t/` (with the host `pyconfig.h`) and +`BUILD-INFO.txt` (Emscripten version, flags, whether the trampoline patch was applied). The +driver has no install or package step for Emscripten, so the script assembles it. + +## Choices made for this experiment + +These are agent proposals from the #175 investigation, not decisions: + +- **Emscripten**: the version CPython itself pins (`emscripten-version` in + `Platforms/emscripten/config.toml`, 4.0.19 for 3.15.0), not the 5.0.3 that + `build-cpython.sh` pins for `pyemscripten_2026_0`. Which version the library should use is + open. The dispatch input `emscripten_version` overrides it, so 5.0.3 can be tried without + editing the script. +- **Node**: 24, the version `config.toml` pins, passed as `--host-runner node`. +- **Flags**: + + ```sh + python3 Platforms/emscripten configure-build-python -- --disable-gil + python3 Platforms/emscripten make-build-python + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-libffi + EMCC_CFLAGS=-pthread python3 Platforms/emscripten make-mpdec + python3 Platforms/emscripten configure-host --clean --host-runner node -- \ + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread" \ + "LDFLAGS=-fwasm-exceptions -sSUPPORT_LONGJMP=wasm -pthread -sPTHREAD_POOL_SIZE=8 -sMAXIMUM_MEMORY=2GB" \ + --disable-gil --enable-wasm-pthreads + python3 Platforms/emscripten make-host + ``` + + The driver puts `CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2` before user arguments and + autoconf keeps the last assignment, so the full string is repeated. `--enable-wasm-pthreads` + makes configure add `-pthread`, `-sUSE_PTHREADS` and `-sPROXY_TO_PTHREAD` itself. + `-sPTHREAD_POOL_SIZE=8` pre-creates Workers, because `Thread.start()` waits for the new + thread and a Worker cannot start while the JS thread that requested it is blocked. + `-sMAXIMUM_MEMORY=2GB` gives the shared memory a maximum, which shared memory requires. +- **libffi and mpdec** are rebuilt with `-pthread` through `EMCC_CFLAGS` (emcc appends it to + every invocation), because the driver builds them without it and wasm-ld is expected to + refuse non-atomic objects in a shared-memory link. +- The build Python is free-threaded too, as in [`free-threaded-mobile.md`](free-threaded-mobile.md). + +## The one source patch + +`Python/emscripten_trampoline.c` instantiates a wasm-gc call trampoline in pre-run. That +module is compiled with `-Wl,--import-memory` and no `-pthread` (`Makefile.pre.in`), so it +imports a non-shared memory, and instantiating it against the shared memory of a pthread +build is expected to throw outside the `try` block, at startup. Pyodide #237 reports this on +3.14, plus a second problem: the trampoline's `addFunction` pointer exists only in the main +thread's table. `build-wasm.sh` adds an early `return 0` to `getPyEMTrampolinePtr()` when +`wasmMemory.buffer` is a `SharedArrayBuffer`, which selects CPython's existing JS trampoline +(`wasmTable.get(func)` on the calling thread). The edit is asserted, so an upstream change +fails the build instead of skipping the patch. The dispatch input `trampoline_patch: skip` +builds without it, to observe the unpatched failure. + +No patch is applied for `--disable-gil` itself; the investigation found none needed. + +## Scripts + +`.github/scripts/ft-wasm/`: + +- `build-wasm.sh [VERSION]`: unpacks `Python-.tgz` (through + `.github/scripts/ft-mobile/common.sh`), installs Emscripten into `$FT_WORK/emsdk`, applies + the patch, runs the steps above, verifies, assembles and re-verifies the archive, and + writes its sha256 to the job summary. Honours `FT_WORK`, `FT_OUT` (default under `.tmp/`), + `FT_WASM_EMSDK_VERSION` and `FT_WASM_TRAMPOLINE_PATCH`. +- `smoke.py`: run under node by the script, see below. + +The build is meant for CI (`AGENTS.md` rule 10). The workflow runs on `workflow_dispatch` +and on pushes to `develop` that change its own files. + +## Verification built into the workflow + +The script stops at the first failed check, in this order: + +1. `emcc --version` reports the requested Emscripten version. +2. The build Python reports `Py_GIL_DISABLED == 1`. +3. `libffi.a` and `libmpdec.a` carry `+atomics` in their wasm `target_features`. +4. The host `pyconfig.h` has `#define Py_GIL_DISABLED 1`, the host Makefile has + `-sUSE_PTHREADS` in `CONFIGURE_LDFLAGS_NODIST` and `ABIFLAGS` is `t`. +5. `make-host` links `python.wasm` and runs `python.sh --version` (a startup failure, for + example the trampoline, fails here). +6. `python.wasm` has a shared memory with a declared maximum (read from its import or memory + section; the page counts are logged). +7. `smoke.py` under `python.sh` in the build tree: `sys.platform == "emscripten"`, + `Py_GIL_DISABLED == 1`, `sys._is_gil_enabled() is False`, then four `threading.Thread`s + each compute a sum; all must join within 120 s with the right results and four distinct + thread idents, and the GIL must still be off. It prints `FT-WASM-SMOKE-OK ...`, which the + script greps for. The run has a 600 s timeout (exit 124 means a thread start may have + deadlocked). +8. The interpreter's own `sysconfig.get_path("stdlib")` ends in `python3.15t`; the archive's + stdlib directory is named from it. +9. The unpacked archive has `include/python3.15t/pyconfig.h` with `Py_GIL_DISABLED 1`, + `lib/python3.15t/os.py` and `_sysconfigdata_*.py`, and `smoke.py` passes again with + `PYTHONHOME` pointing at it. + +The archive is uploaded as the workflow artifact `python-t-wasm32-emscripten`. No +release is published or changed. + +## Not verified + +- The workflow has not run yet. Whether 3.15.0 configures, compiles and links with + `--disable-gil --enable-wasm-pthreads` on Emscripten 4.0.19 is unknown. The only real + evidence is upstream's free-threaded build **without** pthreads (CPython gh-158997, on + `main` with Emscripten 6.0.10) and Pyodide's **GIL** build with pthreads (3.14, Emscripten + 5.0.3). No project is known to have combined the two. +- Whether `EMCC_CFLAGS=-pthread` is enough for libffi and mpdec, and whether the + `-sUSE_BZIP2` port links into a shared-memory build, is a prediction. +- Whether the JS trampoline fallback is enough, and what it costs per call compared with the + wasm-gc trampoline (`WasmCallTrampolineTest` covers only the current GIL build). +- Only node is exercised; no browser (which also needs cross-origin isolation for + `SharedArrayBuffer`). +- The smoke test checks that four threads run and join with the GIL off. It is not a + thread-safety test, does not run CPython's test suite, and does not measure parallel + speed-up. +- mimalloc's WASI path on Emscripten has no thread-exit hook; the investigation predicts a + slow per-thread leak. Not measured. + +## Open: the Kotlin side (later steps of #175, not implemented) + +- Kotlin/Wasm imports `intrinsics.memory` as non-shared (`min=0, max=none`); a shared + memory cannot satisfy that import, so the library cannot instantiate this `python.wasm` + until the import is marked shared with a maximum. That would revive the binary patch + [`wasm-design.md`](wasm-design.md) calls obsolete. +- Kotlin calls CPython on the JS thread that loaded the module (`noInitialRun`, no `main`), + so `-sPROXY_TO_PTHREAD` does not apply there; a first `Thread.start()` from Kotlin relies on + the pre-created worker pool. +- Upcalls from a non-main Python thread: the Kotlin trampolines are placed with `table.set` + only in the main thread's table; each Worker has its own table, so every Worker needs the + same entries (SPEC U-5, the pythonx-concurrent wasm dispatchers). +- Staging this archive as a wasmJs runtime (it lacks `wasmExports,wasmMemory` in + `-sEXPORTED_RUNTIME_METHODS` and the `pyemscripten` platform tag that `build-cpython.sh` + adds). diff --git a/docs/platforms/wasm-design.md b/docs/platforms/wasm-design.md index d0d385609..cd1eb3720 100644 --- a/docs/platforms/wasm-design.md +++ b/docs/platforms/wasm-design.md @@ -125,6 +125,10 @@ Details and the postcondition that guards the entry-module ordering are in the w - Whether compiled wheels other than pydantic-core load. - Compose (`@Composable` from Python) on wasm is `planned` (SPEC B-6, N-2); `suspend` upcalls are `planned` on wasm because there are no threads (SPEC U-5). +- Threads (#175): a free-threaded CPython 3.15t build with pthreads is a CI experiment + (`.github/workflows/build-ft-wasm.yml`), separate from the build above and not used by the library. + What it checks, and the Kotlin-side work it does not do (shared memory import, worker pool, + per-Worker function tables), is in [`free-threaded-wasm.md`](free-threaded-wasm.md). - Absolute wasmJs nanoseconds depend on suite scope and host lifetime (a filtered run is 55–74% slower in pure-Python rows); quote ratios, or full-suite figures labelled as such ([`downcall-design.md`](../design/downcall-design.md), "Warmup"). From f0e74d30e805ed3c0472e2f00c54d9e5fdf6370d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:52:08 +0900 Subject: [PATCH 41/61] TypedPython: drop the ob_mutex scheme; pure functions read 3+ lists element by element (#255) Leader decision 2026-11-13: public critical-section API only, no ob_mutex. With that, no scheme locks three or more lists exactly (design 4.3.6 gives the evidence per candidate), so functions that write to 3+ lists stay interpreted as on develop. A pure function reads each element with PyList_GetItemRef in program order, takes no lock and copies nothing (verify/array-direct). nbody compiles energy; the 1-2 list path generates the same C as develop. --- docs/SPEC.md | 4 +- docs/design/typedpython.md | 222 +++++---- .../src/main/python/typedpython/cgen.py | 145 +++--- .../src/main/python/typedpython/frontend.py | 19 +- .../src/main/python/typedpython/ir.py | 16 + .../src/main/python/typedpython/resume.py | 6 +- .../main/python/typedpython/runtime/API.md | 49 +- .../python/typedpython/runtime/tp_runtime.h | 206 +++------ .../src/main/python/typedpython/verify.py | 71 ++- .../src/test/python/test_class_methods.py | 6 +- .../src/test/python/test_free_threaded.py | 427 +++++++++++------- .../src/test/python/test_frontend.py | 3 +- 12 files changed, 591 insertions(+), 583 deletions(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 573302097..7e2545bc7 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -740,7 +740,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `continue` and `return` meaning exactly what they mean without chunking (trip counts just below, at and above a multiple of the interval, negative steps, empty ranges and ranges spanning the i64 limits all behave as CPython's `range`). A function holding an array copy-in or entry-globals snapshot polls too (#152, option (i)): at the top of a loop iteration, once per `TP_POLL_INTERVAL`, it writes its copied lists back (CPython's own stores would already be visible there), ends their critical sections, polls, takes the critical sections again, copies the lists in again and re-checks the entry guards (list lengths, exact element types, entry globals). Unchanged: it continues compiled, which is an execution CPython can produce, since every list operation of another thread now falls before or after this point. Changed: it leaves for the interpreter at that point (an OSR-style exit) and does not deopt or raise: a continuation function generated from the function's own source, holding the rest of the function from that loop iteration (the innermost loop from its current index; each enclosing loop's remaining body, then its remaining iterations; then the statements after), receives every live local (a native value as its Python object, a local array as a new list) and each enclosing `range`'s already evaluated bounds, and the compiled call returns what it returns or raises what it raises. The continuation keeps the original file, line numbers and function name, so a traceback reads as the interpreted function's. - A function with list parameters holds the lists' locks from its copy-in to its write-back, so every list operation of another thread that takes the list's lock (every store, resize, `sort`, ...) falls before or after the call (or a poll above): one or two lists through CPython's critical section (#159), three or more through every list's own lock (#255; agent proposal, not confirmed: design §4.3.6), taken without waiting while another of them is held. When they cannot all be taken (two of them held by one critical section of the calling thread itself, e.g. a `list.sort` key) the call runs interpreted, before any effect; a poll that cannot take them again leaves for the interpreter as when a list changed. Nothing in such a call may wait while the locks are held, so a function with three or more list parameters and an entry global or a redo Call in its call tree stays interpreted (`verify/array-locks`; nbody's `offset_momentum`, which reads `SOLAR_MASS`). + List parameters (#159, #255): a function with one or two holds the lists' critical section from its copy-in to its write-back, so every list operation of another thread that takes the list's lock falls before or after the call (or a poll above). A function with an effect and more than two list parameters stays interpreted (CPython's critical sections lock one or two objects; no other lock is used, leader decision 2026-11-13). A pure function with three or more (`ir.direct_lists`) does not copy them in and takes no lock: each `a[i]` is one `PyList_GetItemRef`, the interpreter's own read, in program order, a negative index counting from the length at that read, `len(a)` the length at that point; an element of another type deopts (the call runs interpreted, nothing having happened). Such a function has no entry global, no buffer parameter, no `a[:]` of a list parameter and no parallel loop (`verify/array-direct`). nbody: `energy` compiles this way; `advance` and `offset_momentum` stay interpreted. **Floating-point exactness and `@compiled(fastmath=True)` (#153, #332, owner decision 2026-10-25):** by default (`fastmath=False`), floating-point compilation is bit-identical to CPython (`#pragma STDC FP_CONTRACT OFF` in the generated C, `-ffp-contract=off -fno-fast-math` on GCC/Clang, `/fp:precise` @@ -777,7 +777,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. | `array.array[float]` / `array.array[int]` parameters (exact `array.array`, typecode `d` or `q`/`l`, 8-byte items) are read and written in place: the buffer export is held for the call and released on every exit, and every element access is one relaxed atomic 8-byte load or store, so each access is atomic like the interpreter's `a[i]`; no copy-in and no limit on the number of buffers. While the export is held, a resize of that array (`append`, `extend`, `del a[i]`, ... from another thread) raises BufferError, exactly as CPython does for any held export; the compiled call never sees a resize. Concurrent writes from another thread to the same elements are not synchronised: as in CPython, no ordering between the two threads' element accesses is guaranteed, only that each access is whole (no torn element). `memoryview[float]` / `memoryview[int]` parameters (#149: exact `memoryview`, 1-D, C-contiguous, format `d` or `q`/`l`, 8-byte aligned, writable when stored to; any exporter: an `array.array`, a numpy array, a `bytearray` cast with `.cast('d')`) take the same path with array.array's element semantics (an element is a Python float / int); the export is taken on the memoryview, so while the call runs `release()` on it raises BufferError, as CPython does for an exported view. An out-of-range index raises memoryview's `IndexError("index out of bounds on dimension 1")`. A released, read-only (when stored to), non-contiguous, multi-dimensional or other-format view deopts: the interpreter raises or runs it. numpy arrays: see the next row | anything else: object path | | numpy `npt.NDArray[np.float64]` / `NDArray[np.int64]` parameters (exact `numpy.ndarray`, 1-D, C-contiguous, native byte order, writable when stored to) take the same buffer path (#220, step 1) when the function accesses the array purely by element (`x[i]`, `len(x)`); a function that uses it as a whole object (`np.sum(x)`, `x.T`, passing `x` on, a slice) compiles as before with the parameter an opaque object. `NDArray[np.bool_]` (#261) is the same path with 1-byte items (buffer format `?`; a `uint8` or other array deopts): an element read is a native bool, boxed as `np.bool` when it escapes; a store compiles only for a bool value (`x[i] = v`, `v: bool`), because numpy converts an int or float by truthiness and that function stays interpreted. Other dtypes are opaque objects. Results follow the interpreter: a value derived from an element is a numpy scalar there, so it is boxed as `np.float64` / `np.int64` / `np.bool` when it escapes (return, `Box`), decided per value at run time by a taint flag; a function declared `-> float` / `int` / `bool` that holds a numpy array returns the interpreter's object. `float(v)` of such a value is a plain float. An out-of-range index raises numpy's `IndexError("index i is out of bounds for axis 0 with size n")`. Arithmetic on a value derived from an element follows numpy's scalar semantics (#220, step 2): float `/`, `//`, `%` never raise ZeroDivisionError (inf / nan, with numpy's result bits and sign rules, `//` and `%` as numpy's `divmod`), int64 `//` and `%` by zero give 0, int64 `/` converts both operands to double. numpy's RuntimeWarning (category and text exactly as numpy's, e.g. `overflow encountered in scalar add`, `divide by zero encountered in scalar divide`, `invalid value encountered in scalar floor_divide`) is emitted once per flagged operation, with the source file and line of the operation. A pure function does not emit it itself: on any floating-point exception (overflow, invalid, divide by zero) or int64 wrap or division by zero it deopts, so the interpreter emits it; an impure function cannot deopt after an effect, so the compiled code emits the warning (`PyErr_WarnExplicitObject`, honouring `warnings` filters and the module's `__warningregistry__`). Float `+ - * / // %` on such values is compiled in both kinds of function, including a store of the result into the array (`x[i] -= d * m`); int64 `/ // %` is compiled in a pure function only. Int64 `+ - *` and `// %` of an element in an impure function (a native INT would have to be a numpy scalar) stay on the object path | anything else: object path | | A function holding an array parameter runs no user code: closed, and no slot read or `C(args)` whose slow path runs user code, in its whole call tree (#203) | not compiled; diagnostic | - | A function with three or more list parameters waits for nothing while it holds their locks: no entry global and no redo Call in its whole call tree (#255) | not compiled; diagnostic (`verify/array-locks`) | + | More than two list parameters only in a pure function, which reads them element by element: no entry global, no buffer parameter, no CopyArray of a list parameter, no parallel loop, no proven (unchecked) index into them (#255) | not compiled; diagnostic (`verify/structure`, `verify/array-direct`) | | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | | Integer overflow checked and promoted (deopt); an op marked `proven` must re-prove its interval fits i64 | not compiled; diagnostic `verify/proven-overflow` | | `Call.redo` only from an impure caller to a pure, may-deopt callee returning F64/BOOL/NONE | not compiled; diagnostic `verify/redo` | diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 4d375c14c..80fc3575b 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -467,137 +467,125 @@ i64/f64 지역)는 **컴파일된 구간 안의 임시 값**만 줄인다. 컨 - 효과가 작아 하지 않는 것: `-OO`(0.35~0.8 MB, `assert` 도 사라짐), deopt 원본 지연 로딩(함수 1000 개당 약 0.4 MB), 동결 모듈 끄기(오히려 4.4 MB 증가). -### 4.3.6 리스트 매개변수 세 개 이상: 리스트마다 잠금 (#255, 조사와 제안) +### 4.3.6 리스트 매개변수 세 개 이상 (#255, 조사 기록과 제안) -**출처.** 이슈 #255 본문: "User decision (2026-10-17): research option (C) for the nbody regression from #209." -같은 본문: "Until (C) is settled, (B) stays." 오너가 정한 것은 (C) 를 **조사한다**는 것이고, 아래의 방식 -(C') 과 그 조건은 **에이전트 제안, 확인되지 않음**이다. +**출처.** +- 이슈 #255 본문: "User decision (2026-10-17): research option (C) for the nbody regression from #209." + 같은 본문: "Until (C) is settled, (B) stays." +- **리더 결정 (2026-11-13, 리더, 코디네이터 경유):** #255 는 리스트의 `ob_mutex` 에 기대지 않는다(critical + section API 전용으로 예약된 필드). 공개 critical section API 만 쓴다. +- 아래의 후보 판정과 (d1) 의 채택은 **에이전트 제안, 확인되지 않음**이다. **문제.** #209 이후 리스트 매개변수가 셋 이상인 함수는 3.15t 에서 해석 실행된다. copy-in 부터 write-back 까지 모든 리스트의 잠금을 쥐어야 하는데, 공개 critical section API 는 객체 하나(`PyCriticalSection`) 또는 둘 (`PyCriticalSection2`)만 잠근다. nbody 의 `offset_momentum`(리스트 4개), `advance`·`energy`(7개)가 31배에서 -1배로 떨어졌다. (A) copy-in 과 write-back 때만 잠그기는 그 사이 다른 스레드의 원소 쓰기를 잃고, (B) 지금처럼 -거절하기가 현재 상태다. +1배로 떨어졌다. 기준은 동일성이다: 컴파일된 실행 하나하나가 CPython 의 어떤 interleaving 과 같아야 한다. #### 가. CPython 3.15.0rc1 의 근거 -venv 의 3.15.0rc1t 헤더와, python/cpython 의 태그 `v3.15.0rc1` 소스(설치된 `pycore_lock.h` 와 바이트 단위로 -같음을 확인)에서 읽었다. 줄 번호는 그 태그 기준이다. +venv 의 3.15.0rc1t 헤더와 python/cpython 의 태그 `v3.15.0rc1` 소스(설치된 `pycore_lock.h` 와 바이트 단위로 같음을 +확인)에서 읽었다. 줄 번호는 그 태그 기준이다. 1. **Begin.** 빠른 경로는 mutex 를 CAS 로 한 번 잡고 스레드의 섹션 스택에 올린다 (`Include/internal/pycore_critical_section.h:106-117`). 실패하면 `_PyCriticalSection_BeginSlow` - (`Python/critical_section.c:20-62`): 같은 mutex 를 **맨 위** 섹션이 이미 쥐었으면 다시 잠그지 않고 돌아간다 - (29-45), world 가 멈춰 있으면 잠그지 않는다(46-54), 아니면 **새 섹션을 먼저 스택에 올리고** `PyMutex_Lock` - 으로 기다린다(55-60). 두 객체는 주소 순으로 잡고(`pycore_critical_section.h:163-170`), 느린 경로는 첫 mutex 를 - 쥔 채 둘째를 기다린다(`critical_section.c:91-101`). -2. **Detach 와 중단.** `PyMutex_Lock` 은 기다릴 때 스레드 상태를 놓는다(`Python/lock.c:654-656`, 138-139; - `Python/parking_lot.c:328-337`). 놓을 때 `detach_thread` 가 `_PyCriticalSection_SuspendAll` 을 불러 - (`Python/pystate.c:2285-2296`) **활성 섹션 전부**의 mutex 를 푼다(`critical_section.c:106-130`). 다시 붙을 때는 - **맨 위 섹션 하나만** 다시 잡는다(`pystate.c:2269-2273`, `critical_section.c:132-165`). 그래서 세 번째 섹션을 - 중첩하면 경합 때 바깥 둘이 풀린 채로 남는다(`Include/critical_section.h:16-22`: "This API does not provide a - way to lock more than two objects at once"). -3. **Stop-the-world.** `stop_the_world` 는 놓인(detached) 스레드를 SUSPENDED 로 바꾸고 붙어 있는 스레드에는 - eval breaker 비트만 세운다(`pystate.c:2385-2408`, 2409-2470). 기다리는 스레드는 놓인 상태이므로 GC 를 막지 - 않는다. world 가 멈춘 동안의 Begin 은 잠그지 않는다(`critical_section.c:46-54`, 69-74). GC 는 할당에서 예약만 - 하고(`Python/gc_free_threading.c:2031-2035`, 2696-2702) eval breaker 에서 돈다. -4. **`ob_mutex` 의 지위.** `PyObject` 의 필드다(`Include/object.h:162`, `Py_GIL_DISABLED` 일 때만; - `_Py_OPAQUE_PYOBJECT`, 곧 abi3t 에서는 숨겨진다). 문서는 "reserved for use by the critical section API ... Do - not lock it directly with `PyMutex_Lock`; doing so can cause deadlocks" 라고 한다 - (`Doc/c-api/structures.rst:58-66`, `Doc/howto/free-threading-extensions.rst:391-410`). `PyMutex` 의 비트(0b01 - 잠김, 0b10 대기자 있음)는 공개 헤더에 "not part of the public API, but described" 로 적혀 있다 - (`Include/cpython/pylock.h:26-35`). 한편 인터프리터 자신이 `ob_mutex` 를 섹션 밖에서 **시도만** 하고 실패하면 - 일반 경로로 간다: `LOCK_OBJECT` = `PyMutex_LockFast(&op->ob_mutex)` (`Python/ceval_macros.h:318-319`), - `STORE_SUBSCR_LIST_INT` (`Python/bytecodes.c:1431`), `list.append` 특수화(5114). 3.15.0rc1 에서 그 밖의 직접 - 사용은 faulthandler 덤프의 시간 제한 0 시도(`Python/pylifecycle.c:3517-3540`)와 섹션 API 를 거치는 - `typeobject.c:83` 뿐이다(`grep ob_mutex`). -5. **섹션을 여는 CPython 연산.** `PyList_SetItem` 은 리스트의 섹션을 연다(`Objects/listobject.c:454-479`). - dict 읽기는 실패 경로에서 dict 의 섹션을 연다(`Objects/dictobject.c:1647-1660`). `list.extend(dict)` 는 - `PyCriticalSection2(list, dict)` 다(`listobject.c:1489`). `list.sort` 는 key 를 부르는 동안 리스트의 섹션을 - 쥔다(`listobject.c:2925` `@critical_section`, 2993 의 key 호출). -6. **잠그지 않는 읽기.** 3.15 의 리스트 원소 읽기는 잠그지 않는다(`listobject.c:354-379`, `bytecodes.c:1148-1151`). - -#### 나. 판정 - -- **`PyMutex_Lock(&list->ob_mutex)` 를 주소 순으로 (원안 C): 건전하지 않다.** 두 가지가 깨진다. - (1) 자기 교착: 호출자가 그 리스트의 섹션을 쥔 채 우리를 부르면(`lst.sort(key=lambda v: f(lst, b, c))`) - `PyMutex_Lock` 이 기다리며 놓을 때 SuspendAll 이 그 섹션을 풀고, 다시 붙을 때 Resume 이 같은 mutex 를 잡으려 한다. - 공정성 handoff 로 mutex 가 이미 우리 몫이 되었으면 Resume 의 `PyMutex_Lock` 은 영원히 기다린다(그 안에서 다시 - 놓아도 맨 위 섹션이 비활성이라 풀 것이 없다, `critical_section.c:113`). handoff 가 없으면 깨어날 때마다 Resume 이 그 mutex 를 다시 잡아 우리 시도가 또 실패하는 일을 되풀이한다. 문서의 경고가 이 경우다. - (2) 잠근 채 기다림: 앞 리스트를 직접 쥔 채 다음 리스트를 기다리면 SuspendAll 은 직접 쥔 것을 풀지 않고, 다시 붙을 - 때 Resume 이 호출자 섹션의 mutex 를 주소 순서와 무관하게 잡는다. 주소 순 체인이 깨져 순환이 생길 수 있다. -- **공개 API 만으로: 같은 보장을 얻는 방법을 찾지 못했다.** 섹션을 중첩한 뒤 하나도 중단되지 않았는지 확인하려면 - 섹션 태그(`_cs_prev`, 비공개 필드)를 읽어야 한다. 내용 비교(double collect)는 리스트에 버전 번호가 없어 ABA 를 - 배제하지 못한다. `PyCriticalSection*_BeginMutex`(3.14 부터 공개, `Include/cpython/critical_section.h:38-42`)도 - mutex 둘까지다. 이 판단이 놓칠 수 있는 것: 3.15 이후에 다객체 섹션 API 가 생기면 이 절은 그 API 로 바꾼다. -- **제안 (C'), 에이전트 제안, 확인되지 않음.** `tp_cs_begin_n` (tp_runtime.h). 리스트를 주소 순으로 정렬하고 - 라운드마다 - 1. 리스트 하나(`w`)에 **CPython 섹션**을 연다. 기다려야 하면 CPython 규칙대로 기다린다(위 1, 2: 호출자 섹션은 - 기다리는 동안 풀리고, 맨 위 섹션이 쥔 리스트는 다시 잠그지 않는다); - 2. 나머지 리스트의 `ob_mutex` 는 **시도만** 한다(`_PyMutex_LockTimed` 의 첫 CAS, `lock.c:58-62`, 대기자 비트 유지); - 3. 하나라도 잡혀 있으면 직접 잡은 것을 풀고 섹션을 닫고, 다음 라운드는 그 리스트로 1 을 한다. 64 라운드 - (`TP_CS_ROUNDS`) 안에 못 가지면 아무것도 쥐지 않은 채 deopt 한다(효과 전, 해석 실행). - write-back 은 `PyList_SetItem` 대신 그 본문(색인 확인, release store, 옛 항목 해제)을 섹션 없이 한다 - (`tp_*_array_exit_held`): `PyList_SetItem` 은 직접 쥔 리스트에 섹션을 열어 자기 스레드를 기다린다(위 5). - 리스트 하나, 둘은 바뀌지 않는다(`PyCriticalSection`, `PyCriticalSection2`). -- **조건 (verifier `verify/array-locks`, cgen 도 다시 거절).** 리스트가 셋 이상인 함수는 호출 트리 전체에 - entry global 도 redo Call 도 없어야 한다. entry global 읽기는 dict 섹션을 기다릴 수 있고(위 5), redo Call 은 - 해석 실행이다. 잠금을 쥔 창 안에서 그런 대기가 생기면, 예를 들어 다른 스레드가 `lst.extend(globals())` 로 dict - 를 먼저 쥐고 `lst` 를 기다리는 동안 우리는 `lst` 를 쥐고 dict 를 기다린다. closed (#203) 는 이미 요구된다. - -#### 다. 증명한 것과 가정한 것 - -증명(위 소스에 근거한 논증): - -- **배제.** 다른 스레드의 섹션과 `LOCK_OBJECT` 는 같은 mutex 비트를 보므로, 우리가 쥔 리스트에서 기다리거나 - (`critical_section.c:59`) 일반 경로로 간다(`bytecodes.c:1431`). 잠그는 리스트 연산은 모두 호출 앞이나 뒤에 놓인다. -- **교착 없음.** 이 스레드는 mutex 를 직접 쥔 채로는 기다리지 않는다: 직접 잡기는 시도뿐이고, 기다림은 직접 쥔 것을 - 모두 푼 뒤 섹션 하나로만 한다. 창 안에는 기다리는 연산이 없다(조건 + closed + write-back 직접 저장). 그러니 창 - 동안 이 스레드에서 나가는 대기 간선이 없어 순환에 들 수 없다. 끝낼 때는 직접 쥔 것을 먼저 풀고 섹션을 닫는다 - (닫기가 바깥 섹션을 다시 잡으며 기다릴 수 있기 때문, `pycore_critical_section.h:128-137`). -- **섹션이 창 동안 중단되지 않음.** 중단은 detach(`pystate.c:2290-2292`) 와 기다리는 Begin 에서만 일어나고, 창 - 안에는 둘 다 없다. -- **재진입.** 호출자의 맨 위 섹션이 쥔 리스트는 그 섹션이 계속 쥔다(Begin 이 다시 잠그지 않음, 위 1). 아래 섹션이 - 쥔 리스트는 우리가 기다리는 동안 풀려 우리 것이 된다(CPython 의 중첩 의미 그대로). 맨 위의 - `PyCriticalSection2` 하나가 두 리스트를 쥔 경우만 영원히 못 가지며, 64 라운드 뒤 deopt 로 끝난다. -- **#152 의 성질 유지.** 긴 루프의 poll 은 write-back 하고 모든 잠금을 푼 뒤 poll 하고 다시 잡는다. 다시 못 잡으면 - 리스트가 바뀐 때처럼 해석 실행으로 넘어간다(OSR 출구). 창 안에서 detach 하지 않으므로 GC stop-the-world 는 그 - 호출(또는 다음 poll)을 기다린다: 1~2 리스트와 같은 liveness 지연이다. -- **GIL 빌드.** 모두 no-op 이다(`tp_cs_begin_n` 은 0 을 돌려준다). 두 헤더 집합(3.15t, 3.15)에 대해 - `clang -fsyntax-only` 로 생성 C 를 확인했다. - -가정(증명 아님): - -- **A1.** `ob_mutex` 의 위치와 `PyMutex` 비트 의미가 3.15 동안 바뀌지 않는다. 문서는 직접 사용을 금한다(위 4). - 우리는 문서가 경고하는 `PyMutex_Lock` 은 쓰지 않고, 인터프리터의 `LOCK_OBJECT` 와 같은 시도만 쓴다. 그래도 - "reserved" 를 넘는 사용이므로 **오너 결정이 필요하다.** -- **A2.** 창 안의 C 코드(복사, malloc, float/int 생성과 해제, 예외 객체 생성)는 기다리지 않는다. 소스 감사에 - 근거한 가정이고, 3.15 판이 바뀌면 다시 본다. -- **A3.** CPython 의 다른 코드는 `ob_mutex` 를 섹션 API 밖에서 기다리며 잡지 않는다(위 4 의 grep 결과). - -이 변경 범위 밖에서 찾은 것(1~2 리스트 경로에도 #255 이전부터 있는 성질, 별도 이슈 제안): - -- 1~2 리스트 함수가 창 안에서 entry global 을 읽다가 dict 섹션을 기다리면(위 5) 리스트의 섹션이 중단되어, 그 사이 - 다른 스레드의 쓰기를 write-back 이 덮어쓸 수 있다. redo Call 도 같다. -- 리스트 원소 읽기는 잠그지 않으므로(위 6) write-back 도중의 다른 스레드 읽기는 일부만 쓰인 상태를 볼 수 있다. - "호출 하나가 원자 단계" 는 잠그는 연산에 대해서만 성립한다. - -#### 라. 시험과 결과 - -- `test_free_threaded.py`: 리스트 3개·7개 함수가 컴파일되고 인터프리터와 같은 결과(빈 리스트, 별칭, 리스트가 아닌 - 인자 포함); 리스트 3개를 네 스레드가 서로 다른 순서와 1·2 리스트 함수로 동시에 더해도 증가를 하나도 잃지 않음; - `list.sort` 의 key 에서 부르는 경우(맨 위 섹션, 아래 섹션, 가운데 매개변수) 교착 없이 인터프리터와 같음; - entry global 이 있는 3 리스트 함수는 해석 실행. 변형 시험(`race_mutant`, CI 의 기본 실행 밖): 옵션 (A) - (copy-in 과 write-back 때만 잠금)는 같은 동시 시험에서 증가를 잃어야 한다. -- nbody: `advance`, `energy` 가 컴파일되고 `offset_momentum` 은 `SOLAR_MASS` (entry global) 때문에 해석 실행이다 - (`test_class_methods.py` 의 EXPECTED). 측정은 CI 의 결과를 기다린다. + (`Python/critical_section.c:20-62`): 같은 mutex 를 **맨 위** 섹션이 이미 쥐었으면 다시 잠그지 않고(29-45), + world 가 멈춰 있으면 잠그지 않고(46-54), 아니면 새 섹션을 먼저 올리고 `PyMutex_Lock` 으로 기다린다(55-60). 두 + 객체는 주소 순으로 잡고(`pycore_critical_section.h:163-170`), 느린 경로는 첫 mutex 를 쥔 채 둘째를 기다린다 + (`critical_section.c:91-101`). +2. **Detach 와 중단.** `PyMutex_Lock` 은 기다릴 때 스레드 상태를 놓는다(`Python/lock.c:654-656`; + `Python/parking_lot.c:328-337`). 놓을 때 활성 섹션 전부를 풀고(`Python/pystate.c:2285-2296`, + `critical_section.c:106-130`), 다시 붙을 때 **맨 위 하나만** 다시 잡는다(`pystate.c:2269-2273`, + `critical_section.c:132-165`). 그래서 세 번째 섹션을 중첩하면 경합 때 바깥 둘이 풀린 채 남는다 + (`Include/critical_section.h:16-22`: "This API does not provide a way to lock more than two objects at once"). +3. **섹션 구조체는 비공개.** "the contents of the structures are private" + (`Doc/c-api/synchronization.rst:225-232`, `Include/critical_section.h:39-40`). 스레드의 섹션 스택 + (`PyThreadState.critical_section`)을 읽는 공개 함수는 없다. `PyMutex_IsLocked` 는 잠겼는지만 알려 주고 누가 + 쥐었는지는 모른다. 3.14 의 `PyCriticalSection*_BeginMutex` 도 mutex 둘까지다 + (`Include/cpython/critical_section.h:38-42`). +4. **`ob_mutex`.** 문서: "reserved for use by the critical section API ... Do not lock it directly with + `PyMutex_Lock`" (`Doc/c-api/structures.rst:58-66`, `Doc/howto/free-threading-extensions.rst:391-410`). +5. **잠그는 CPython 연산과 잠그지 않는 읽기.** `PyList_SetItem` 은 리스트의 섹션을 연다 + (`Objects/listobject.c:454-479`). `list + list` 는 두 리스트를 함께 잠근다(`listobject.c:813`). `list.sort` 는 + key 를 부르는 동안 리스트의 섹션을 쥐고 리스트를 비워 둔다(`listobject.c:2925`, 2993). 원소 읽기 + `PyList_GetItemRef` 는 잠그지 않는 원자적 읽기이고(`listobject.c:354-379`, 405-419), 인터프리터의 `a[i]` 가 바로 + 그 읽기다(`Python/bytecodes.c:1148-1151`). dict 읽기는 실패 경로에서 dict 의 섹션을 연다 + (`Objects/dictobject.c:1647-1660`). `len` 은 원자적 길이 읽기다(3.15t `Include/cpython/listobject.h:30-36`). + +#### 나. 앞선 조사: `ob_mutex` 를 쓰는 방안 (리더 결정으로 제외) + +- **원안 (C), `PyMutex_Lock(&list->ob_mutex)` 를 주소 순으로: 건전하지 않다.** 호출자 섹션이 쥔 리스트 + (`lst.sort(key=lambda v: f(lst, b, c))`)에서 자기 교착한다: 기다리며 놓을 때 SuspendAll 이 그 섹션을 풀고, 다시 + 붙을 때 Resume 이 같은 mutex 를 잡으려 한다. 공정성 handoff 로 mutex 가 이미 우리 몫이면 그 Resume 은 영원히 + 기다린다(`critical_section.c:113`: 비활성 맨 위 섹션은 다시 풀지 않는다). 또 직접 쥔 채 기다리면 Resume 이 호출자 + 섹션의 mutex 를 주소 순서와 무관하게 잡아 순환이 생길 수 있고, write-back 의 `PyList_SetItem` 이 자기 스레드를 + 기다린다. +- **변형 (C'), 섹션 하나 + 나머지는 `ob_mutex` 시도만 (커밋 89e9bfae):** 위 결함은 피했지만 `ob_mutex` 를 문서의 + "reserved" 를 넘어 쓴다. 리더 결정(2026-11-13)으로 제외했고 코드에서 지웠다(`ob_mutex`, `PyMutex` 비트를 직접 쓰는 + 곳이 남아 있지 않다). + +#### 다. 공개 API 만으로 본 후보와 판정 + +- **(a) copy-in 전에 섹션을 중첩해 잡고, 하나도 중단되지 않았음을 확인: 불가.** 확인하려면 섹션 태그(`_cs_prev` + 의 INACTIVE 비트)나 스레드의 섹션 스택을 읽어야 하는데 둘 다 비공개다(위 3). 중단은 우리가 기다리는 순간(Begin + 의 느린 경로)에 일어나므로 시점을 피할 수도 없다(위 1, 2). +- **(b) 낙관적 검증: 리스트마다 자기 섹션 안에서 copy-in 하고 읽은 항목의 강한 참조를 쥔 뒤, 커밋 때 읽은 칸의 + 동일성을 다시 보고 write-back: 일반적으로 건전하지 않다.** + - 커밋이 원자적이지 않다. 섹션은 둘까지라, 세 리스트 이상에 쓰는 함수의 write-back 은 리스트별로 나뉜다. 예: + `a[0] = 1; b[0] = 1; a[0] = 2`. 인터프리터에서 다른 스레드가 볼 수 있는 상태는 (0,0), (1,0), (1,1), (2,1) 이다. + a 를 먼저 커밋하면 (2,0) 이, b 를 먼저 커밋하면 (0,1) 이 보인다. 둘 다 CPython 이 만들지 않는 상태이고, + `a + b`(두 리스트를 함께 잠근다, 위 5)나 2 리스트 컴파일 함수가 그것을 관측한다. 어떤 커밋 순서도 이것을 막지 + 못한다. 쓰는 리스트가 둘 이하면 `PyCriticalSection2` 하나로 원자적으로 커밋할 수 있다. + - 읽기 검증에는 한 순간이 필요하다. 쓰기와 프로그램 순서로 섞인 읽기는 커밋 순간의 값이어야 하는데, 커밋에서 + 잠글 수 없는 리스트는 그 순간에 볼 수 없다. 커밋 전과 후에 같은 객체를 다시 보는 것(double collect)은 ABA 를 + 막지 못한다: 다른 스레드가 `t = r[0]; r[0] = 5.0; r[0] = t` 로 같은 객체를 되돌릴 수 있고, 리스트에는 공개 + 버전 번호가 없다. + - 그러니 (b) 가 건전한 경우는 쓰기 구간 동안 손대는 리스트가 둘 이하인 경우뿐이고, 그것은 (c) 의 특수 경우다. +- **(c) 컴파일 타임에 리스트를 읽기 전용/쓰기 전용으로 나누거나 함수를 나눠 한 번에 섹션이 둘 이하가 되게: 쓰는 + 함수에는 지금 기반으로 불가.** + - 읽기 전용이라도 여러 리스트를 한 순간에 copy-in 할 필요는 남는다. 반례: 다른 스레드가 `a[1] = 1` 다음 + `b[0] = 1` 을 하고, 프로그램이 `a[0]`, `b[0]`, `a[1]` 순으로 읽는다. a 를 t1 에, b 를 t2 > t1 에 따로 copy-in 하면 + (`b[0]` = 1, `a[1]` = 0) 을 볼 수 있는데, 인터프리터는 `b[0]` 에서 1 을 보면 뒤의 `a[1]` 에서도 1 을 본다. + - 함수를 프로그램 지점에서 구간으로 나누고(각 구간이 리스트 둘 이하), 경계에서 write-back 하고 다시 copy-in 하는 + 것은 #152 의 poll 과 같은 논리로 건전하다. 하지만 경계에서 리스트가 바뀌었으면(길이, 원소 타입) 그 지점에서 + 인터프리터로 넘어가야 하고, 효과 뒤에는 deopt 할 수 없다. 지금 OSR 출구는 루프 poll 지점에만 있다. 임의 + 문장 지점의 연속(continuation)이 생기기 전에는 쓰는 함수에 쓸 수 없다(nbody `advance` 는 안쪽 루프 한 바퀴가 + 일곱 리스트를 모두 건드린다). +- **(d1) 순수 함수: copy-in 없이 원소마다 공개 API 로 읽기: 건전하다 (채택, 에이전트 제안, 확인되지 않음).** + - 순수 함수는 리스트를 읽기만 한다. 각 `a[i]` 를 `PyList_GetItemRef`(인터프리터의 읽기 그 자체, 원자적, 위 5) + 한 번으로, IR 의 프로그램 순서대로(문장마다 임시 변수에 순서대로 emit) 읽는다. 음수 색인은 그 읽기 때의 길이로 + 센다(인터프리터와 같음). `len(a)` 는 그 지점의 길이다. 이 실행은 인터프리터의 읽기를 같은 순서로 하는 것이므로 + 다른 스레드의 연산이 읽기 사이 어디에 끼든 그 interleaving 과 같다. 잠금을 쥐지 않으므로 교착도, 호출자 섹션 + 문제도, stop-the-world 지연도 없다. + - 원소 타입이 다르면(다른 스레드가 int 를 넣었다) deopt 한다. 순수 함수는 아직 아무 효과도 내지 않았으므로 + 해석 실행으로 다시 하는 것이 그대로 한 실행이다. 리스트가 줄었으면 그 지점에서 인터프리터와 같은 + `IndexError("list index out of range")`. 길이가 호출 중 바뀔 수 있으므로 증명된(검사 없는) 색인은 쓰지 않는다. + - 조건(verifier `verify/array-direct`, cgen 도 다시 거절): entry global 없음(입구에서 한 번 읽은 전역과 나중의 원소 + 읽기가 어떤 순서와도 안 맞을 수 있다, 위 (c) 의 반례와 같은 꼴), 버퍼 매개변수 없음, 리스트 매개변수의 + `a[:]` 없음(한 순간의 사본이 필요), 병렬 루프 없음(작업자는 Python API 를 쓰지 않는다). + - 리스트 하나, 둘의 경로는 그대로다: 같은 입력에 develop 과 바이트 단위로 같은 C 를 만든다(생성 C 비교로 확인). + - **비용(측정 전):** 읽기마다 함수 호출 하나와 참조 증감 한 쌍이다. copy-in 경로보다 느리고, 얼마나 빠른지는 CI + 측정을 기다린다. +- **(d2) 쓰는 함수의 원소별 접근: 불가(지금).** 쓰기는 `PyList_SetItem` 하나로 원자적이지만, 효과 뒤의 읽기가 다른 + 타입을 만나면 인터프리터는 계속 실행하는데 우리는 그 지점의 연속이 없다(위 (c) 와 같은 한계). 임의 지점 OSR 출구가 + 생기면 (d2) 가 nbody `advance` 를 푼다. + +#### 라. 판정과 결과 + +- 셋 이상의 리스트에 **쓰는** 함수는 (B) 그대로 해석 실행이다(develop 과 같은 frontend/verify/cgen 거절). +- **순수** 함수는 (d1) 로 컴파일된다. nbody: `energy` 가 컴파일되고, `advance`·`offset_momentum` 은 해석 + 실행이다(`test_class_methods.py` 의 EXPECTED). nbody 전체 속도는 `advance` 가 정하므로 거의 그대로일 것이다(측정 전). +- 시험(`test_free_threaded.py`): 순수 3·7 리스트 함수의 컴파일, 인터프리터와 같은 결과(빈 리스트, 별칭, 다른 타입 + 원소의 deopt, 리스트가 아닌 인자, 짧은 리스트의 IndexError); 쓰는 3 리스트 함수와 entry global 이 있는 3 리스트 + 함수는 해석 실행; 프로그램 순서 시험(쓰는 스레드가 `c[0]` 다음 `a[0]` 을 쓰고, `a[0]`, `c[0]` 순으로 읽는 함수가 + 늘 `c[0] >= a[0]` 을 본다); 동시에 int 와 float 를 바꿔 넣어도 결과가 늘 인터프리터의 것; `list.sort` key 에서 + 부르기. 변형 시험: 읽기 순서를 바꾼 변형(`race_mutant`, CI 기본 실행 밖)과 원소 타입 검사를 뺀 런타임(결정적, + CI 에서 실행)이 잡혀야 한다. 옵션 (A) 변형은 3 리스트 쓰기 경로가 없어져 대상이 없으므로 지웠다. #### 마. 오너 결정이 필요한 점 -1. (C') 를 받아들일지: `ob_mutex` 를 문서의 "reserved" 를 넘어 시도로 잡는다(A1). 받아들이지 않으면 (B) 로 - 돌린다(frontend/verify/cgen 의 조건 하나). -2. entry global 이 있는 3 리스트 이상 함수(nbody `offset_momentum`): 잠그기 전에 읽고 copy-in 뒤 전역 epoch 로 - 재검증하는 방안이 있으나 메모리 순서 논증이 끝나지 않았다. 지금은 해석 실행. -3. `TP_CS_ROUNDS` = 64 (에이전트 제안). +1. (d1) 을 받아들일지(에이전트 제안). 받아들이지 않으면 순수 함수도 (B) 로 돌린다(`ir.direct_lists` 하나). +2. 쓰는 함수(nbody `advance`)를 위해 임의 문장 지점의 OSR 출구를 만들지: (c) 의 구간 나누기와 (d2) 가 둘 다 그것에 + 걸려 있다. ### 4.4 Kotlin 경계의 직접 호출 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index ae30cb1aa..10baaa1a2 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -25,15 +25,14 @@ Free-threaded CPython 3.15t (issue #159; runtime/API.md "Free-threaded CPython" holds the table of every piece of shared state). The module slots declare `{Py_mod_gil, Py_MOD_GIL_NOT_USED}` (under `#ifdef Py_GIL_DISABLED`), so importing it keeps the GIL off. In `tp_wrap_f`, a function with list -parameters holds the lists' locks from before the first copy-in to after the last write-back: -`tp_cs_end` is part of the same `exits` that run the write-back on every path (ok, error, deopt, the -last before the interpreted redo), and is a no-op on a path that left before locking. One list or -two: CPython's critical section (`tp_cs_begin`). Three or more (#255, design/typedpython.md 4.3.6): -`tp_cs_begin_n` holds every list's own mutex; when it cannot take them (a list locked by this very -thread's caller) the call deopts before any effect, and the write-back is `tp_*_array_exit_held`. -Nothing in that window may wait, so a function with three or more lists and an entry global or a -redo Call in its call tree is refused here (`CGenError`), as in the verifier (verify/array-locks). -A function that holds an +parameters holds the lists' critical section (`tp_cs_begin`, one list or two) from before the first +copy-in to after the last write-back: `tp_cs_end` is part of the same `exits` that run the write-back +on every path (ok, error, deopt, the last before the interpreted redo), and is a no-op on a path +that left before locking. More than two list parameters are refused here (`CGenError`), as in the +front end and the verifier: CPython has no three-object critical section; except in a pure function +(#255, `ir.direct_lists`): its lists are not copied, every Index is `tp_list_ld_*` +(`PyList_GetItemRef` in program order, an element of another type deopts), every Len the current +length, and no lock is taken. A function that holds an array does not poll the eval breaker (#141, open decision #152), so a long compiled array loop delays other threads' stop-the-world requests until it returns: a liveness delay, results unaffected. The `__typedpython_deopts__` read-add-write of `tp_cg_redo` runs in the module dict's @@ -925,11 +924,11 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: if f.entry_globals: _check_entry_globals(f, functions) n_arrays = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params) - if n_arrays > 2: - why = _lock_window_waits(f, functions, set()) - if why is not None: - raise CGenError(f"{f.name}: {n_arrays} list parameters hold every list's lock from copy-in " - f"to write-back, so nothing in the call may wait, but {why} (#255)") + if ir.direct_lists(f): + _check_direct_lists(f) + elif n_arrays > 2: + raise CGenError(f"{f.name}: {n_arrays} array parameters; at most two (CPython offers one- and " + "two-object critical sections only, and a third lock could suspend the others)") for p in f.params: if isinstance(p, ir.ArrayParam): if p.type not in ir.ARRAYS: @@ -965,24 +964,24 @@ def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> return True -def _lock_window_waits(f: ir.Function, functions: dict[str, ir.Function], seen: set[str]) -> str | None: - """Why [f] or a function it calls can wait (an entry global read can take the module dict's - critical section; a redo Call runs the interpreted callee), or None (#255).""" - if f.name in seen: - return None - seen.add(f.name) - if f.entry_globals: - return f"'{f.name}' reads entry global '{f.entry_globals[0].name}'" - for e in _all_exprs(f): - if isinstance(e, ir.Call): - if e.redo: - return f"'{f.name}' has a redo Call of '{e.function}'" - callee = functions.get(e.function) - if callee is not None: - why = _lock_window_waits(callee, functions, seen) - if why is not None: - return why - return None +def _check_direct_lists(f: ir.Function) -> None: + """#255 (ir.direct_lists): what a pure function with three or more lists read element by element + may not have.""" + why = None + if not f.may_deopt: + why = "may_deopt is False (every element read can deopt)" + elif f.entry_globals: + why = f"entry global {f.entry_globals[0].name} (it would be read once at entry)" + elif any(isinstance(p, ir.ArrayParam) and (p.buffer is not None or p.stored + or p.type not in _ARRAY_PFX) for p in f.params): + why = "a buffer parameter, a stored list or a list of another element type" + elif any(isinstance(e, ir.CopyArray) and any(q.name == e.src for q in f.params) for e in _all_exprs(f)): + why = "a CopyArray of a list parameter" + elif any(isinstance(s, ir.ForRange) and s.par is not None for s in _walk_stmts(f.body)): + why = "a parallel loop" + if why is not None: + raise CGenError(f"{f.name}: three or more list parameters are read element by element (#255), " + f"which rules out {why}") def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> None: @@ -1104,6 +1103,9 @@ def __init__(self, mg: _ModGen, f: ir.Function, fast: bool = False, rmode: str | self.depth = 1 self.temps: list[tuple[str, str]] = [] # (C type, name) self.arrays = {p.name: p for p in f.params if isinstance(p, ir.ArrayParam)} + # #255: a pure function with three or more lists reads them element by element (no copy-in) + self.direct = {p.name for p in f.params if isinstance(p, ir.ArrayParam) and p.buffer is None} \ + if ir.direct_lists(f) else set() self.par_defs: list[str] = [] # outlined region bodies (SPEC N-15), emitted before the impl self.local_arrays: dict[str, Type] = {} # name -> F64_ARRAY / I64_ARRAY self.brk: list = [] # per open loop: goto label for `break`, or None @@ -1984,6 +1986,9 @@ def _array(self, name: str) -> tuple[Type, str]: def ev_Len(self, e: ir.Len) -> str: _, ptr = self._array(e.array) t = self.tmp("int64_t") + if e.array in self.direct: + self.emit(f"{t} = (int64_t)PyList_GET_SIZE({ptr}->list); /* #255: the current length */") + return t self.emit(f"{t} = (int64_t){ptr}->len;") return t @@ -2010,6 +2015,17 @@ def ev_Index(self, e: ir.Index) -> str: at, ptr = self._array(e.array) if e.type is not _ARRAY_ELEM[at]: raise CGenError(f"{self.f.name}: Index of {at} typed {e.type}") + if e.array in self.direct: + # #255: one PyList_GetItemRef (the interpreter's read), in program order; IndexError, or a + # deopt for an element of another type + if e.index.type is not Type.I64: + raise CGenError(f"{self.f.name}: index of type {e.index.type}") + i = self.ev(e.index) + t = self.tmp(_CT[e.type]) + sfx = "f64" if at is Type.F64_ARRAY else "i64" + self.emit(f"tp_s = tp_list_ld_{sfx}({ptr}->list, {i}, &{t});") + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t buffer = self._buffer(e.array) slot = self._slot(at, ptr, e.index, e.proven, 0, buffer) t = self.tmp(_CT[e.type]) @@ -2267,7 +2283,8 @@ def st_If(self, s: ir.If) -> None: @property def snapshot(self) -> bool: """Holds an array copy-in or an entry-globals snapshot, so must not run user code (#141).""" - return bool(self.f.entry_globals) or any(isinstance(p, ir.ArrayParam) for p in self.f.params) + return bool(self.f.entry_globals) or any(isinstance(p, ir.ArrayParam) and p.name not in self.direct + for p in self.f.params) def poll(self, every_time: bool = False) -> None: """At the top of each loop iteration (#141): signals, pending calls, GIL hand-off. @@ -2364,15 +2381,13 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None [site] (the impl returns 2 with tp_my->result set). Only the call its wrapper entered polls.""" r = self.osr_sites_by_site[site] lists = self._list_params() - held = len(lists) > 2 # #255: every list's mutex, not a critical section - ex = "_exit_held" if held else "_exit" self.emit("if (tp_my != NULL) { /* #152: poll inside a snapshot holder */") self.depth += 1 self.emit("int tp_och = 0;") for j, p in enumerate(lists): ptr = _ident("l", p.name) self.emit(f"Py_ssize_t tp_ol{j} = {ptr}->len;") - self.emit(f"if ({_ARRAY_PFX[p.type]}{ex}({ptr}) < 0) {{ tp_rc = -1; goto tp_exit; }}") + self.emit(f"if ({_ARRAY_PFX[p.type]}_exit({ptr}) < 0) {{ tp_rc = -1; goto tp_exit; }}") if lists: self.emit("tp_cs_end(tp_my->lock);") self.emit("tp_t->snapshot_depth--;") @@ -2387,22 +2402,13 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None self.emit("#endif") self.emit("tp_t->snapshot_depth++;") self.emit("if (tp_s != 0) { tp_rc = -1; goto tp_exit; }") - if held: - # not taken again (a list locked by this thread's caller): leave for the continuation with - # nothing held (tp_och 2: no fresh copies). One statement per line: the race mutant of - # test_free_threaded.py drops every line that names tp_cs_begin. - self.emit("tp_s = 0;") - self.emit(f"tp_s = tp_cs_begin_n(tp_my->lock, tp_my->lock->objs, {len(lists)});") - self.emit("if (tp_s != 0) tp_och = 2;") - elif lists: - second = "tp_my->lists[1]" if len(lists) == 2 else "NULL" - self.emit(f"tp_cs_begin(tp_my->lock, tp_my->lists[0], {second});") + if lists: + self.emit("tp_cs_begin(tp_my->lock, tp_my->lists[0], tp_my->lists[1]);") for j, p in enumerate(lists): ptr = _ident("l", p.name) - guard = "if (tp_och != 2) " if held else "" - self.emit(f"{guard}{{ tp_s = {_ARRAY_PFX[p.type]}_enter(tp_my->lists[{j}], {ptr});") - self.emit(" if (tp_s < 0) { tp_cs_end(tp_my->lock); tp_rc = -1; goto tp_exit; }") - self.emit(f" if (tp_s != 0 || {ptr}->len != tp_ol{j}) tp_och = 1; }}") + self.emit(f"tp_s = {_ARRAY_PFX[p.type]}_enter(tp_my->lists[{j}], {ptr});") + self.emit("if (tp_s < 0) { tp_cs_end(tp_my->lock); tp_rc = -1; goto tp_exit; }") + self.emit(f"if (tp_s != 0 || {ptr}->len != tp_ol{j}) tp_och = 1;") for g in self.f.entry_globals: k = self.mg.name_index(g.name) lv = _ident("l", g.name) @@ -2417,8 +2423,8 @@ def snap_poll(self, site: int, cur: str, stop: str = "0", step: int = 0) -> None self.emit("if (tp_och) { /* leave for the interpreter at this iteration (OSR-style exit) */") self.depth += 1 for j, p in enumerate(lists): - # fresh copies hold no dirty element: this exit only frees them (a zeroed struct: no-op) - self.emit(f"(void){_ARRAY_PFX[p.type]}{ex}({_ident('l', p.name)});") + # fresh copies hold no dirty element: this exit only frees them + self.emit(f"(void){_ARRAY_PFX[p.type]}_exit({_ident('l', p.name)});") if lists: self.emit("tp_cs_end(tp_my->lock);") n = len(r.params) @@ -2989,13 +2995,15 @@ def wrapper(self) -> str: args = [] arrays = [] # list parameters: copy-in / write-back under a critical section buffers = [] # buffer parameters: a Py_buffer export held for the whole call (#149) - if any(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params): + direct = [] # #255: lists read element by element (ir.direct_lists), no copy-in, no lock + is_direct = ir.direct_lists(f) + if any(isinstance(p, ir.ArrayParam) and p.buffer is None for p in f.params) and not is_direct: L.append(" tp_cs tp_lock = {0};") for i, p in enumerate(f.params): if isinstance(p, ir.ArrayParam): L.append(f" {'tp_buf' if p.buffer else _ARRAY_CT[p.type]} tp_a{i} = {{0}};") args.append(f"&tp_a{i}") - (buffers if p.buffer else arrays).append((i, p)) + (buffers if p.buffer else direct if is_direct else arrays).append((i, p)) elif p.type is Type.OBJ: args.append(f"tp_args[{i}]") # borrowed for the call's duration else: @@ -3005,9 +3013,7 @@ def wrapper(self) -> str: args.append("&tp_r") # PyBuffer_Release first (cannot fail), then the lists' write-back and the critical section exits = [f" tp_buf_exit(&tp_a{i});" for i, p in buffers] - held = len(arrays) > 2 # #255: every list's mutex is held directly (tp_cs_begin_n) - ex = "_exit_held" if held else "_exit" - exits += [f" if ({_ARRAY_PFX[p.type]}{ex}(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] + exits += [f" if ({_ARRAY_PFX[p.type]}_exit(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] if arrays: exits.append(" tp_cs_end(&tp_lock); /* after the write-back; a no-op if never taken */") has_exits = bool(arrays or buffers) @@ -3056,6 +3062,13 @@ def wrapper(self) -> str: f"{fmt1}, {fmt2}, {int(p.stored)}, &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") L.append(" if (tp_s != 0) goto tp_deopt;") + if direct: + # #255: exact lists (a subclass may override __getitem__ / __len__), not copied; the + # wrapper's arguments keep them alive for the call + L.append(" if (" + " || ".join(f"!PyList_CheckExact(tp_args[{i}])" for i, _ in direct) + + ") goto tp_deopt;") + for i, _ in direct: + L.append(f" tp_a{i}.list = tp_args[{i}]; /* borrowed; read by tp_list_ld_* */") # aliasing guard, then copy-in if len(arrays) > 1: objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) @@ -3067,14 +3080,8 @@ def wrapper(self) -> str: # as tp_*_array_enter would, without being locked. L.append(" if (" + " || ".join(f"!PyList_CheckExact(tp_args[{i}])" for i, _ in arrays) + ") goto tp_deopt;") - if held: - # the lists in address order (sorted in place by tp_cs_begin_n), kept for the polls - objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) - L.append(f" PyObject *tp_lobjs[{len(arrays)}] = {{{objs}}};") - L.append(f" if (tp_cs_begin_n(&tp_lock, tp_lobjs, {len(arrays)}) != 0) goto tp_deopt;") - else: - second = f"tp_args[{arrays[1][0]}]" if len(arrays) == 2 else "NULL" - L.append(f" tp_cs_begin(&tp_lock, tp_args[{arrays[0][0]}], {second});") + second = f"tp_args[{arrays[1][0]}]" if len(arrays) == 2 else "NULL" + L.append(f" tp_cs_begin(&tp_lock, tp_args[{arrays[0][0]}], {second});") for i, p in arrays: L.append(f" tp_s = {_ARRAY_PFX[p.type]}_enter(tp_args[{i}], &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") @@ -3102,12 +3109,10 @@ def wrapper(self) -> str: osr = self.osr if osr: lst = [i for i, p in arrays] + l0 = f"tp_args[{lst[0]}]" if lst else "NULL" + l1 = f"tp_args[{lst[1]}]" if len(lst) > 1 else "NULL" lock = "&tp_lock" if lst else "NULL" - if lst: - # the list arguments in parameter order (tp_lobjs, the lock's own array, is sorted) - L.append(f" PyObject *const tp_olists[{len(lst)}] = {{" - + ", ".join(f"tp_args[{i}]" for i in lst) + "};") - L.append(f" tp_osr tp_ox = {{{lock}, {'tp_olists' if lst else 'NULL'}, NULL}};") + L.append(f" tp_osr tp_ox = {{{lock}, {{{l0}, {l1}}}, NULL}};") L.append(" tp_t->osr = &tp_ox; /* #152: taken by the impl at entry */") L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module, tp_ts, tp_t{''.join(', ' + a for a in args)});") if osr: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 8f81d2ca1..1c3c2332e 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -743,9 +743,6 @@ def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = No params.append(ir.Param(a.arg, Type.OBJ, c[0], c[1])) else: params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) - # Any number of list parameters (#255): one or two are held by CPython's critical section, three - # or more by every list's own lock (cgen.py, design/typedpython.md 4.3.6); the verifier refuses the - # latter when something in the call could wait (verify/array-locks). if fn.returns is None: raise Skip(line, "no return annotation") if isinstance(fn.returns, ast.Constant) and fn.returns.value is None: @@ -1074,6 +1071,22 @@ def run(self) -> ir.Function: pure = not self.impure assert pure or not self.deopts or not self.impure_mode, \ "impure mode emitted a deopting node" + n_lists = sum(isinstance(p, ir.ArrayParam) and p.buffer is None for p in params) + if n_lists > 2: + # Free-threaded CPython (#159): an impure function's lists are locked from copy-in to + # write-back, and CPython offers one- and two-object critical sections only (a nested + # third could suspend the others). A pure one reads them element by element (#255, + # ir.direct_lists), and a read of an element of another type deopts. + if not pure: + raise Skip(self.fn.lineno, f"more than two list parameters ({n_lists}) are not supported " + "in a function with an effect: compiled code holds each list's " + "lock from copy-in to write-back, and free-threaded CPython has " + "no lock for more than two objects at once") + if self.entry_used: + raise Skip(self.fn.lineno, f"more than two list parameters ({n_lists}) and a module global " + "read: the lists are read element by element, so the global " + "cannot be read once at entry (#255)") + self.deopts = True fn = ir.Function( name=self.sig.qname, params=params, returns=self.sig.returns, locals=locals_, body=body, pure=pure, may_deopt=self.deopts, source_line=self.fn.lineno, diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 4b28c068f..f29cc38be 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -886,6 +886,22 @@ class Module: REGION_SCALAR_KINDS = {Type.I64: "i64", Type.F64: "f64", Type.BOOL: "bool"} +def direct_lists(f: object) -> bool: + """#255 (design/typedpython.md 4.3.6): a pure function with three or more list parameters reads + them element by element, without copy-in or critical section. CPython's critical sections lock + one or two objects, and copying three or more lists in at one instant needs them all locked (the + leader decision of 2026-11-13 excludes any lock outside that API). A pure function only reads + its lists, so each `Index` is one `PyList_GetItemRef` in program order, the interpreter's own + atomic read, and an element of the wrong type deopts (legal: nothing has happened yet); `Len` is + the current length. No entry global (a snapshot read at entry could disagree with later list + reads), no buffer parameter, no CopyArray of a list parameter and no parallel loop: the verifier + (`verify/array-direct`) and cgen refuse them. An impure function with more than two list + parameters stays interpreted (#209).""" + params = getattr(f, "params", ()) + lists = sum(isinstance(p, ArrayParam) and p.buffer is None for p in params) + return getattr(f, "pure", False) is True and lists > 2 + + def region_arg_kind(a: object, ref_class) -> object: """The region kind one `New` argument gives its field: "i64" / "f64" / "bool" for a `Box` of that scalar, "null" for the `None` constant, ("ref", class) for a value of a region class (`ref_class(a)` diff --git a/python-multiplatform-ksp/src/main/python/typedpython/resume.py b/python-multiplatform-ksp/src/main/python/typedpython/resume.py index 0a87c5db2..de419c3c3 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/resume.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/resume.py @@ -54,7 +54,11 @@ def __tp_resume_def(src, lmap): def has_snapshot(f: ir.Function) -> bool: """Same rule as `cgen._FnGen.snapshot` (cgen.py, property `snapshot`, ~line 2186): any - entry global, or any ArrayParam (buffer or not; the property does not look at `buffer`).""" + entry global, or any ArrayParam (buffer or not; the property does not look at `buffer`), except + the lists of a function that reads them element by element (`ir.direct_lists`, #255).""" + if ir.direct_lists(f): + return bool(f.entry_globals) or any(isinstance(p, ir.ArrayParam) and p.buffer is not None + for p in f.params) return bool(f.entry_globals) or any(isinstance(p, ir.ArrayParam) for p in f.params) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 3b0d9faa3..4deb3819b 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -73,31 +73,25 @@ Python error set, nothing to undo), `-1` Python error set (CPython's exception t /* aliasing guard: 1 if any two of the n objects are the same object */ int tp_any_same(PyObject *const *objs, Py_ssize_t n); - /* Free-threaded CPython (#159): the wrapper holds the lists' locks from before the first enter - to after the last exit, on every path (ok, error, deopt). tp_cs is zero-initialised - (`tp_cs c = {0};`). tp_cs_begin: b == NULL locks one list, else both (PyCriticalSection2: - ordered by address, so two threads locking (a, b) and (b, a) cannot deadlock). - tp_cs_begin_n (#255; agent proposal, not confirmed): three or more distinct exact lists; - sorts objs by address in place (objs stays the lock's, for the re-take after a poll) and holds - one list through a critical section and the others through their own mutex (ob_mutex), never - blocking while it holds a mutex directly; 0 = all held, 1 = not taken (nothing held: the - wrapper deopts, a poll leaves for the continuation) after TP_CS_ROUNDS (64) rounds, which - happens when two of the lists are held by this thread's own top-most PyCriticalSection2. With - a GIL all three are no-ops (tp_cs_begin_n returns 0). */ - typedef struct { int n, w; PyObject **objs; /* + a PyCriticalSection2 on free-threaded builds */ } tp_cs; + /* Free-threaded CPython (#159): the wrapper holds the lists' critical section from before the + first enter to after the last exit, on every path (ok, error, deopt). tp_cs is zero-initialised + (`tp_cs c = {0};`). b == NULL locks one list, else both (PyCriticalSection2: ordered by + address, so two threads locking (a, b) and (b, a) cannot deadlock). More than two list + parameters are not compiled (frontend.Skip, verify/structure, CGenError): CPython has no + three-object section and a nested third would suspend the others. With a GIL both are no-ops. */ + typedef struct { int n; /* + a PyCriticalSection2 on free-threaded builds */ } tp_cs; void tp_cs_begin(tp_cs *c, PyObject *a, PyObject *b); - int tp_cs_begin_n(tp_cs *c, PyObject **objs, int n); void tp_cs_end(tp_cs *c); /* idempotent: a no-op when nothing is held */ - /* exit with three or more lists: the list's mutex is held directly, so the write-back is - PyList_SetItem's store without its critical section (which would wait for this thread) */ - int tp_f64_array_exit_held(tp_f64_array *a); - int tp_i64_array_exit_held(tp_i64_array *a); + /* #255, ir.direct_lists: a pure function with three or more list parameters does not copy them + in and takes no lock (the exception to "more than two ... not compiled" above). The wrapper + checks PyList_CheckExact and sets only `list` of each tp_*_array; every Index is one of these + (PyList_GetItemRef, the interpreter's read; a negative index counts from the length at that + read): 0 = *out set, -1 = IndexError "list index out of range", 1 = deopt (not exactly float / + not exactly an int within i64). Len is PyList_GET_SIZE at that point. */ + int tp_list_ld_f64(PyObject *list, int64_t i, double *out); + int tp_list_ld_i64(PyObject *list, int64_t i, int64_t *out); `tp_*_array_enter` / `_exit` read and write the list through borrowed references (`PyList_GET_ITEM`, -`PyList_SetItem`) and REQUIRE the caller to hold that critical section (`_exit_held`: the list's -mutex, held by `tp_cs_begin_n`). With three or more lists nothing between enter and exit may wait -(a detach or a blocking lock would keep the directly held mutexes while this thread waits): -verify/array-locks refuses an entry global or a redo Call anywhere in the call tree, and cgen -refuses them again (CGenError). Between enter and exit no +`PyList_SetItem`) and REQUIRE the caller to hold that critical section. Between enter and exit no other thread can change the list, so the whole call is one atomic step as far as lists go (an interleaving CPython can produce: each list operation takes the same lock). A long compiled array loop does not poll (below), so it delays other threads' stop-the-world requests (the free-threaded @@ -128,8 +122,8 @@ call by the export. Rules that go with it (verifier / front end): * Closed (#203) like a list: a function with a buffer runs no user code (`verify/array-open`). -* No limit on the number of buffer parameters (no critical section), nor of list parameters since #255 - (three or more: `tp_cs_begin_n`, with the verify/array-locks condition). +* No limit on the number of buffer parameters (no critical section); lists keep the limit of two in a + function with an effect (a pure function reads three or more element by element, #255). A buffer and a list cannot alias (different types). Two buffers may share memory (the same array passed twice): per-element access makes reads see earlier writes in program order, as in the interpreter, which a copy-in could not. @@ -249,8 +243,7 @@ Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryEr Thread state (#125): the wrapper calls `PyThreadState_GetUnchecked()` once and passes it as `tp_ts` to the impl, which passes it to `tp_enter_call`, `tp_depth_room`, `tp_traced` and to its direct compiled callees, so a compiled call and its C-to-C callees do one TLS lookup of the thread state per call. Likewise the per-thread state (call depth, poll countdown, snapshot depth) is one `__thread` struct -`tp_osr` (#152, OSR-style exit): `{ tp_cs *lock; PyObject *const *lists; PyObject *result; }` (`lists`: the -wrapper's array of the list arguments in parameter order, any number since #255). A snapshot-holding +`tp_osr` (#152, OSR-style exit): `{ tp_cs *lock; PyObject *lists[2]; PyObject *result; }`. A snapshot-holding function's wrapper sets `tp_t->osr` to its own context right before calling the impl; the impl takes it at entry and sets `tp_t->osr` back to NULL. The impl returns 2 when it left for the continuation, with `result` set (strong). @@ -499,7 +492,7 @@ static or module-state field must be added here): | `buf_types` (array.array) | module state | immutable after init | written by `tp_exec` (and `tp_clear`); read-only type pointers compared by identity | | buffer arguments of array parameters | caller's array.array memory | atomic | relaxed 8-byte `__atomic_load_n` / `__atomic_store_n` per element; the Py_buffer export is held for the call (open point on concurrent resize, #149) | | list arguments of array parameters | caller's lists | critical section | `tp_cs_begin` before the copy-in, `tp_cs_end` after the write-back, on every path | -| list arguments, three or more (#255) | caller's lists | each list's lock | `tp_cs_begin_n`: one list through a critical section, the others' `ob_mutex` tried without blocking (a taken one ends the round and is waited for through a critical section, nothing held); every lock held from copy-in to write-back, write-back by `_exit_held`; nothing in that window waits (verify/array-locks). Agent proposal, not confirmed (design/typedpython.md 4.3.6) | +| list arguments of a pure function with three or more (#255) | caller's lists | CPython | not copied, no lock: every element read is `PyList_GetItemRef` (atomic, a strong reference), in program order; the wrapper's arguments keep the lists alive | | module globals (class names, entry globals) | module dict | strong reads | `tp_global` and `tp_class_global_ok` use `PyDict_GetItemRef` | | a compiled class's instances | other threads' objects | CPython | `PyMember_GetOne` / `SetOne` are what the interpreter's member descriptors call (they take the object's lock on free-threaded builds); `tp_new_fixed` writes only its own fresh object | | `errno` (math helpers) | C library | thread-local | per thread by POSIX | @@ -513,7 +506,7 @@ a module is kept alive by an own reference while it is read; `PyEval_GetBuiltins `PyEval_GetFrameBuiltins` (a new reference). `tp_class_global_ok`: `PyDict_GetItemWithError` became `PyDict_GetItemRef` inside the dict's critical section. `tp_cg_redo`: the counter update is under the module dict's critical section. `PyList_GET_ITEM` in `tp_*_array_enter` stays a borrow: it is read -under the list's critical section the caller holds (with three or more lists, the list's lock). Borrows that cannot race: `PyModule_GetDict` (the +under the list's critical section the caller holds. Borrows that cannot race: `PyModule_GetDict` (the dict lives as long as the module, which the running function keeps alive), `PyTuple_GET_SIZE` of a kwnames tuple, `PyTuple` items. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 4b909d9ae..37f4b8a2f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -112,42 +112,11 @@ static inline unsigned int tp_type_tag(const PyTypeObject *cls) #define TP_THREAD_LOCAL __thread #endif -/* Locks for list parameters (tp_cs_*): from before the copy-in to after the write-back, on every - * exit path. With a GIL they are no-ops. - * - * One list or two: CPython's own critical section (PyCriticalSection / PyCriticalSection2, ordered - * by address). Three or more (#255, design/typedpython.md 4.3.6; agent proposal, not confirmed): - * CPython has no section for more than two objects, and a nested third may suspend the outer ones - * (Include/critical_section.h), so tp_cs_begin_n holds one list through a critical section and the - * others through their own mutex (ob_mutex): - * - it never blocks while it holds a mutex directly: each round begins a critical section on one - * list (which waits, if it must, the way CPython requires: this thread's own outer sections are - * suspended while it waits, and a list held by this thread's top-most section is not locked - * again), then only TRIES the other lists' mutexes (the compare-and-swap of PyMutex_LockFast - * and of the interpreter's LOCK_OBJECT). If one is taken it releases what it got, ends the - * section and begins the next round with a critical section on that list; - * - a list that a caller of this thread holds (list.sort holds its list while it calls the key) - * is either the round's own list (the top-most section holds it: not locked again) or is freed - * when the round's section waits (the wait suspends the caller's sections). Two lists held by - * one section of a caller (a PyCriticalSection2 that is top-most) can never both be had, so - * after TP_CS_ROUNDS rounds it gives up with nothing held and the call runs interpreted (a - * deopt, before any effect); contention alone ends a round only when another thread took a - * list between two of its attempts; - * - while they are held nothing may wait (the verifier's verify/array-locks: closed, no entry - * global, no redo Call): the section is not suspended (that needs a detach or a blocking nested - * section), and no thread can wait for this one while this one waits for it; - * - the write-back stores through tp__list_store_held, not PyList_SetItem, which would begin a - * critical section on a directly held list and wait for this very thread (listobject.c). - * Other threads' critical sections, and the interpreter's LOCK_OBJECT fast paths, on these lists - * see the same mutexes locked and wait (or take their locking path), as for a critical section. */ -#ifndef TP_CS_ROUNDS -#define TP_CS_ROUNDS 64 -#endif - +/* Critical sections for array parameters (tp_cs_*): from before the copy-in to after the + * write-back, on every exit path. One list or two (CPython offers one- and two-object sections + * only; a function with more list parameters stays interpreted). With a GIL they are no-ops. */ typedef struct { - int n; /* 0 = not held; 1 or 2: a critical section; >= 3: n lists held */ - int w; /* n >= 3: the index in objs held through the critical section */ - PyObject **objs; /* n >= 3: the lists in address order (the wrapper's array, borrowed) */ + int n; /* 0 = not held, 1 or 2 objects locked */ #ifdef Py_GIL_DISABLED PyCriticalSection2 cs; #endif @@ -170,66 +139,6 @@ static inline void tp_cs_begin(tp_cs *c, PyObject *a, PyObject *b) #endif } -#ifdef Py_GIL_DISABLED -/* one non-blocking attempt: the first step of _PyMutex_LockTimed (Python/lock.c), which keeps the - * "has parked" bit; the bit values are those of Include/cpython/pylock.h */ -static inline int tp__mutex_try(PyMutex *m) -{ - uint8_t v = _Py_atomic_load_uint8_relaxed(&m->_bits); - while ((v & _Py_LOCKED) == 0) { - if (_Py_atomic_compare_exchange_uint8(&m->_bits, &v, (uint8_t)(v | _Py_LOCKED))) { - return 1; - } - } - return 0; -} -#endif - -/* n >= 3 distinct exact lists (the wrapper's aliasing guard ran): sorts objs by address in place and - * takes every list's lock. 0 = all held (end with tp_cs_end); 1 = not taken, nothing held. */ -static inline int tp_cs_begin_n(tp_cs *c, PyObject **objs, int n) -{ - int i, j; - for (i = 1; i < n; i++) { /* the order PyCriticalSection2 uses: lower address first */ - PyObject *o = objs[i]; - for (j = i; j > 0 && (uintptr_t)objs[j - 1] > (uintptr_t)o; j--) { - objs[j] = objs[j - 1]; - } - objs[j] = o; - } - c->objs = objs; -#ifdef Py_GIL_DISABLED - int w = 0; - for (int round = 0; round < TP_CS_ROUNDS; round++) { - int k; - PyCriticalSection_Begin((PyCriticalSection *)&c->cs, objs[w]); - for (k = 0; k < n; k++) { - if (k != w && !tp__mutex_try(&objs[k]->ob_mutex)) { - break; - } - } - if (k == n) { - c->w = w; - c->n = n; - return 0; - } - for (i = k - 1; i >= 0; i--) { - if (i != w) { - PyMutex_Unlock(&objs[i]->ob_mutex); - } - } - PyCriticalSection_End((PyCriticalSection *)&c->cs); - w = k; /* nothing held: the next round waits for the taken one */ - } - c->n = 0; - return 1; -#else - c->w = 0; - c->n = n; - return 0; -#endif -} - /* idempotent: a no-op when nothing is held */ static inline void tp_cs_end(tp_cs *c) { @@ -240,16 +149,6 @@ static inline void tp_cs_end(tp_cs *c) else if (c->n == 2) { PyCriticalSection2_End(&c->cs); } - else if (c->n > 2) { - /* the directly held mutexes first: ending the section may resume an outer one, which can - * wait, and nothing may be held directly then */ - for (int i = c->n - 1; i >= 0; i--) { - if (i != c->w) { - PyMutex_Unlock(&c->objs[i]->ob_mutex); - } - } - PyCriticalSection_End((PyCriticalSection *)&c->cs); - } #endif c->n = 0; } @@ -1101,8 +1000,7 @@ static inline int tp_unbox_bool(PyObject *o, int *out) * enter: rc 0 -> the struct holds the copy; rc 1 or -1 -> the struct is zeroed (so exit is a safe * no-op) and nothing was changed. `list` is borrowed: the caller keeps the argument alive. * - * Free-threaded (#159): the CALLER holds the list's critical section (tp_cs_begin) or, with three - * or more lists, its mutex (tp_cs_begin_n, #255; then it ends with exit_held) from before + * Free-threaded (#159): the CALLER holds the list's critical section (tp_cs_begin) from before * enter to after exit, on every exit path. enter and exit read and write the list's items through * borrowed references (PyList_GET_ITEM) and rely on that lock: no other thread can change the list * between the copy-in and the write-back, so the whole call is one atomic step as far as lists are @@ -1215,36 +1113,7 @@ static inline int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, return tp__array_slot(a->len, i, store, slot); } -/* The store of PyList_SetItem (listobject.c) for a list whose mutex this thread holds directly - * (tp_cs_begin_n): the same index check, release store and decref of the old item, without the - * critical section PyList_SetItem begins, which would wait for this very thread. The old item is an - * exact float or int (the copy-in checked, and the lock was held since), so its decref runs no code. - * Steals v. With a GIL: PyList_SetItem. */ -static inline int tp__list_store_held(PyObject *list, Py_ssize_t i, PyObject *v) -{ -#ifdef Py_GIL_DISABLED - PyListObject *l = (PyListObject *)list; - PyObject *old; - if (i < 0 || i >= Py_SIZE(l)) { - Py_DECREF(v); - PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); - return -1; - } - old = l->ob_item[i]; - _Py_atomic_store_ptr_release(&l->ob_item[i], v); - Py_XDECREF(old); - return 0; -#else - return PyList_SetItem(list, i, v); -#endif -} - -static inline int tp__list_store(PyObject *list, Py_ssize_t i, PyObject *v, int held) -{ - return held ? tp__list_store_held(list, i, v) : PyList_SetItem(list, i, v); -} - -static inline int tp__f64_array_exit(tp_f64_array *a, int held) +static inline int tp_f64_array_exit(tp_f64_array *a) { int rc = 0; if (a->data != NULL && a->dirty != NULL && a->list != NULL) { @@ -1255,7 +1124,7 @@ static inline int tp__f64_array_exit(tp_f64_array *a, int held) continue; } v = PyFloat_FromDouble(a->data[i]); - if (v == NULL || tp__list_store(a->list, i, v, held) < 0) { + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { rc = -1; /* SetItem consumed v on failure; the error is set */ break; } @@ -1270,7 +1139,7 @@ static inline int tp__f64_array_exit(tp_f64_array *a, int held) return rc; } -static inline int tp__i64_array_exit(tp_i64_array *a, int held) +static inline int tp_i64_array_exit(tp_i64_array *a) { int rc = 0; if (a->data != NULL && a->dirty != NULL && a->list != NULL) { @@ -1281,7 +1150,7 @@ static inline int tp__i64_array_exit(tp_i64_array *a, int held) continue; } v = PyLong_FromLongLong((long long)a->data[i]); - if (v == NULL || tp__list_store(a->list, i, v, held) < 0) { + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { rc = -1; break; } @@ -1296,12 +1165,55 @@ static inline int tp__i64_array_exit(tp_i64_array *a, int held) return rc; } -/* exit: the list's critical section is held (one or two lists); exit_held: its mutex is held - * directly (three or more lists, tp_cs_begin_n) */ -static inline int tp_f64_array_exit(tp_f64_array *a) { return tp__f64_array_exit(a, 0); } -static inline int tp_i64_array_exit(tp_i64_array *a) { return tp__i64_array_exit(a, 0); } -static inline int tp_f64_array_exit_held(tp_f64_array *a) { return tp__f64_array_exit(a, 1); } -static inline int tp_i64_array_exit_held(tp_i64_array *a) { return tp__i64_array_exit(a, 1); } +/* ------------------------------------------------------------------------------------------ + * Lists read element by element (#255, ir.direct_lists): a pure function with three or more list + * parameters does not copy them in. The wrapper sets only `list` of each tp_*_array (borrowed; the + * call's arguments keep it alive) and every Index is one of these, in program order: the list's own + * PyList_GetItemRef, which is the read the interpreter's `a[i]` does (Objects/listobject.c + * list_get_item_ref; Python/bytecodes.c _BINARY_OP_SUBSCR_LIST_INT), so each read is atomic and no + * lock is held between two reads. A negative index counts from the length at that read, as in the + * interpreter. rc 0 = *out set; -1 = IndexError("list index out of range"); 1 = deopt (the element + * is not exactly float / not exactly an int within i64; nothing has happened: the function is pure). + * Len is PyList_GET_SIZE at that point (an atomic load on free-threaded builds). + * ------------------------------------------------------------------------------------------ */ +static inline PyObject *tp__list_item_ref(PyObject *list, int64_t i) +{ + if (i < 0) { + i += (int64_t)PyList_GET_SIZE(list); + } + if (i < 0 || i > (int64_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_IndexError, "list index out of range"); + return NULL; + } + return PyList_GetItemRef(list, (Py_ssize_t)i); /* new reference, or NULL + IndexError */ +} + +static inline int tp_list_ld_f64(PyObject *list, int64_t i, double *out) +{ + PyObject *v = tp__list_item_ref(list, i); + if (v == NULL) { + return -1; + } + if (!PyFloat_CheckExact(v)) { + Py_DECREF(v); + return 1; + } + *out = PyFloat_AS_DOUBLE(v); + Py_DECREF(v); + return 0; +} + +static inline int tp_list_ld_i64(PyObject *list, int64_t i, int64_t *out) +{ + int rc; + PyObject *v = tp__list_item_ref(list, i); + if (v == NULL) { + return -1; + } + rc = tp_unbox_i64(v, out); + Py_DECREF(v); + return rc < 0 ? -1 : rc > 0 ? 1 : 0; +} /* ------------------------------------------------------------------------------------------ * Buffer arrays (issue #149; ir.ArrayParam.buffer): an exact array.array argument @@ -3004,7 +2916,7 @@ static inline PyObject *tp_list_from_i64(const int64_t *d, Py_ssize_t n) * returned 2; the wrapper then returns it, with its write-back already done. */ typedef struct tp_osr { tp_cs *lock; /* the wrapper's critical section, or NULL (no list parameter) */ - PyObject *const *lists; /* the list arguments in parameter order (the wrapper's, borrowed), or NULL */ + PyObject *lists[2]; /* the list arguments in critical-section order (borrowed), or NULL */ PyObject *result; /* strong, after an exit: what the continuation returned */ } tp_osr; diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index f76e6d820..b54f31247 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -37,16 +37,17 @@ verify/structure break/continue outside a loop, the ForRange var assigned in its body, a zero or non-constant step, duplicate names, bad declarations, an entry global that collides with a parameter/local/other entry global or - is assigned in the body + is assigned in the body, or more than two array parameters in a + function that is not pure (the lists are locked from copy-in to + write-back: one- and two-object critical sections only, #159) verify/proven-overflow `proven` is set on an op the verifier cannot prove stays inside i64 (or on a node `proven` does not apply to, or it is not a bool) verify/redo `Call.redo` where it is not legal (see below) verify/entry-global-open a function with entry_globals is not closed verify/array-open a function with an ArrayParam runs user code (#203) - verify/array-locks a function with three or more list ArrayParams (#255: it holds every - list's own lock from copy-in to write-back, outside CPython's - critical-section stack, so nothing in that window may wait) has an - entry global or a redo Call, itself or anywhere in its call tree + verify/array-direct a pure function with three or more list parameters (#255, + `ir.direct_lists`: read element by element) has an entry global, a + buffer parameter, a CopyArray of a list parameter or a parallel loop verify/buffer a malformed `ArrayParam.buffer` (None, "array", "numpy", "memoryview"), or a CopyArray of a buffer array (#149) verify/par-loop a `ForRange.par` (SPEC N-15, #277) that the verifier does not recompute to @@ -310,7 +311,7 @@ REDO = "verify/redo" ENTRY_OPEN = "verify/entry-global-open" ARRAY_OPEN = "verify/array-open" -ARRAY_LOCKS = "verify/array-locks" +ARRAY_DIRECT = "verify/array-direct" BUFFER = "verify/buffer" CLASS = "verify/class" FIELD = "verify/field-unproved" @@ -405,6 +406,7 @@ def __init__(self, function: Function, signatures: dict[str, Function], classes: dict[str, ClassDecl] | None = None, var_ivs: dict[str, tuple[int, int]] | None = None, fixing: bool = False): self.fn = function + self.direct = ir.direct_lists(function) # #255: lists read element by element # Local intervals (module docstring "Local variables and guards"): `var_ivs` is the interval # of every eligible local; `fixing` says this walk is a round of the fixpoint that computes # them (a local without an interval yet is then "bottom": `bottom` is set when one is read). @@ -436,7 +438,6 @@ def __init__(self, function: Function, signatures: dict[str, Function], self.entry: dict[str, Type] = {} # name -> type of every entry global self.opens: list[str] = [] # reasons the function is not closed self.slow_user: list[str] = [] # slow paths that can run user code (FieldGet, New) - self.redo_calls: list[str] = [] # callees of the legal redo Calls (#255: they can wait) def error(self, rule: str, message: str) -> None: d = Diagnostic(self.fn.name, rule, message, self.fn.source_line) @@ -473,6 +474,18 @@ def declarations(self) -> frozenset[str]: if p.name in params: self.error(STRUCTURE, f"parameter '{p.name}' appears twice") params.add(p.name) + if self.direct: + self.can_deopt = True # every element read checks the element's type (#255) + if isinstance(fn.entry_globals, tuple) and fn.entry_globals: + self.error(ARRAY_DIRECT, "three or more list parameters are read element by element " + "(#255), so an entry global cannot be read once at entry") + if any(type(p) is ArrayParam and p.buffer is not None for p in fn.params): + self.error(ARRAY_DIRECT, "three or more list parameters are read element by element " + "(#255); a buffer parameter beside them is not supported") + elif sum(type(p) is ArrayParam and p.buffer is None for p in fn.params) > 2: + self.error(STRUCTURE, "more than two array parameters: compiled code holds each list's " + "critical section from copy-in to write-back, and free-threaded " + "CPython has no section for more than two objects (#159)") eg = fn.entry_globals if not isinstance(eg, tuple): self.error(STRUCTURE, "entry_globals is not a tuple") @@ -1071,7 +1084,6 @@ def e_Call(self, e: Call, state: _State) -> ir.Expr: f"(the redone value may be a big int)") else: redo = True - self.redo_calls.append(e.function) if not redo: for p in callee.params: # The callee's entry guard `type(x) is C` may fail, even for an argument that is a @@ -1195,6 +1207,11 @@ def index(self, array: str, index: object, state: _State, node: str def e_Index(self, e: Index, state: _State) -> ir.Expr: idx, element, proven = self.index(e.array, e.index, state, "index") + if self.direct and e.array in self.arrays and e.array not in self.buffers: + # #255: another thread can resize the list between two reads, so no bound is proved, + # and an element of another type deopts + proven = False + self.can_deopt = True if element is not None and e.type is not element and e.type is not Type.OBJ: self.error(TYPE, f"element of '{e.array}' is {element.value}, node says " f"{e.type.value}") @@ -1373,6 +1390,9 @@ def allocation(self, e: NewArray | CopyArray, target: str, state: _State) -> ir. elif e.src in self.buffers: self.error(BUFFER, f"CopyArray of buffer array '{e.src}': a buffer array is never " "copied)") + elif self.direct and e.src in self.arrays: + self.error(ARRAY_DIRECT, f"CopyArray of list parameter '{e.src}' in a function that reads " + "its three or more lists element by element (#255)") else: self.array(e.src, "copy", state) if src is not e.type: @@ -1674,7 +1694,11 @@ def s_ForRange(self, s: ForRange, state: _State): for br in loop.breaks: after = _meet(after, br) par = self._par_info(s, step_ok, body, loop, state) - if s.par is not None and s.par != par: + if self.direct and (par is not None or s.par is not None): + self.error(ARRAY_DIRECT, "a parallel loop in a function that reads its three or more lists " + "element by element (#255)") + par = None + elif s.par is not None and s.par != par: self.error(PAR_LOOP, f"ForRange over '{var}' declares par={s.par!r}, but the verifier " f"computes {par!r} (SPEC N-15)") return dataclasses.replace(s, start=start, stop=stop, body=body, par=par), after @@ -2555,7 +2579,6 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: callees: dict[str, set[str]] = {} open_why: dict[str, str] = {} # a function that runs user code -> why slow_why: dict[str, str] = {} # a function whose slow paths can run user code -> why - wait_why: dict[str, str] = {} # a function that can wait (#255: entry global, redo) -> why for name, g in sigs.items(): c, rebuilt = _prove(g, sigs, classes) for d in c.diags: @@ -2566,12 +2589,6 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: open_why[name] = c.opens[0] if c.slow_user: slow_why[name] = c.slow_user[0] - if isinstance(g.entry_globals, tuple) and g.entry_globals: - wait_why[name] = (f"reads entry global '{getattr(g.entry_globals[0], 'name', '?')}' (the " - "read can take the module dict's critical section)") - elif c.redo_calls: - wait_why[name] = (f"has a redo Call of '{c.redo_calls[0]}' (the interpreted callee can " - "take critical sections)") # Closedness (entry_globals): a fixpoint over the call graph. changed = True @@ -2608,28 +2625,6 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: reject(name, g.source_line, ARRAY_OPEN, f"holds a list copy-in but can run user code: {array_why[name]}") - # Three or more list parameters (#255, design/typedpython.md 4.3.6): the wrapper holds every list's - # own lock (ob_mutex) from the copy-in to the write-back, outside CPython's critical-section stack. - # A detach or a blocking lock in that window would keep those locks while this thread waits, and a - # thread waiting for one of them while holding what this one waits for (PyCriticalSection2 keeps - # its first lock while it waits for the second) would never wake. So nothing in the window may - # wait: closed (above), and no entry global and no redo Call in the whole call tree. - changed = True - while changed: - changed = False - for name, cs in callees.items(): - if name not in wait_why: - bad = sorted(c for c in cs if c in wait_why) - if bad: - wait_why[name] = f"calls '{bad[0]}', which {wait_why[bad[0]]}" - changed = True - for name, g in sigs.items(): - n_lists = sum(isinstance(p, ArrayParam) and p.buffer is None for p in g.params) - if n_lists > 2 and name in wait_why: - reject(name, g.source_line, ARRAY_LOCKS, - f"three or more list parameters ({n_lists}) hold every list's lock from copy-in " - f"to write-back, so nothing in the call may wait, but it {wait_why[name]}") - # A call cycle (direct or mutual recursion) is legal: every compiled impl function counts its # frame at entry (tp_enter_call), so recursion ends in RecursionError where CPython raises it # and the C stack cannot overflow (issue #57). Purity and may_deopt flow through the cycle as diff --git a/python-multiplatform-ksp/src/test/python/test_class_methods.py b/python-multiplatform-ksp/src/test/python/test_class_methods.py index feb440897..2957adb8a 100644 --- a/python-multiplatform-ksp/src/test/python/test_class_methods.py +++ b/python-multiplatform-ksp/src/test/python/test_class_methods.py @@ -920,9 +920,9 @@ def test_binary_trees_with_check_as_a_method_compiles_and_matches(tmp_path): "fannkuch": {"fannkuch"}, "spectral_norm": {"eval_a", "mul_atv", "mul_av"}, "wordfreq": {"next_state", "word_of", "main"}, # word_of and main: native ints, #232 - # #255: three or more lists hold every list's lock; offset_momentum (4 lists) reads the entry - # global SOLAR_MASS, which could wait while they are held (verify/array-locks): interpreted - "nbody": {"advance", "energy"}, + # #255: a pure function reads its three or more lists element by element (energy); advance and + # offset_momentum store into theirs and stay interpreted (no lock for more than two objects) + "nbody": {"energy"}, } diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index fe0d81b66..cdbd52cce 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -5,7 +5,7 @@ * keep its shared mutable state thread-safe (runtime/API.md "Free-threaded CPython"); * read nothing through a borrowed reference that another thread can free; * hold the list's critical section from before the copy-in to after the write-back (array params); - with three or more list parameters, hold every list's lock for that whole window (#255). + a pure function with three or more list parameters reads them element by element instead (#255). Every concurrency scenario runs in a CHILD interpreter with a timeout (a crash or a deadlock fails one test, not the run). Iteration counts scale with TP_FT_STRESS (default 1). @@ -14,9 +14,10 @@ failed (`Py_mod_gil` absent, the GIL came back at import), the concurrent counters lost updates, and a three-list function was compiled. -#255 (three or more lists, design/typedpython.md 4.3.6): red before the change, the functions with three -and seven list parameters (`three`, `bump3`, `seven`) were not compiled, so `built` failed and every test -of this file using it errored; the front-end, verifier and cgen tests of the 3-list section failed. +#255 (three or more lists, design/typedpython.md 4.3.6): red before the change, the pure functions with +three and seven list parameters (`three`, `seven`, `order3`, `sum3`) were not compiled, so `built` failed +and every test of this file using it errored; the front-end, verifier and cgen tests of the 3-list +section failed (verify.ARRAY_DIRECT did not exist). """ from __future__ import annotations @@ -182,13 +183,28 @@ def seven(x: list[float], y: list[float], z: list[float], vx: list[float], vy: l vz: list[float], w: list[float]) -> float: e: float = 0.0 for i in range(len(x)): - vx[i] += x[i] * w[i] - vy[i] -= y[i] * w[i] - vz[i] += z[i] * w[i] * 0.5 - e += vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i] + e += 0.5 * w[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, len(x)): + dx: float = x[i] - x[j] + dy: float = y[i] - y[j] + dz: float = z[i] - z[j] + e -= w[i] * w[j] * (dx * dx + dy * dy + dz * dz) return e +@compiled +def order3(a: list[float], b: list[float], c: list[float]) -> float: + x: float = a[0] + y: float = b[0] + z: float = c[0] + return z - x + 0.0 * y + + +@compiled +def sum3(a: list[float], b: list[float], c: list[float]) -> float: + return a[0] + b[0] + c[0] + + K: int = 3 @@ -242,7 +258,7 @@ def built(tmp_path_factory): assert result.extension is not None, result.skipped compiled = {f.name for f in result.module.functions} wanted = {"geta", "seta", "mk", "total", "fill", "edge", "fill_int", "edge_int", "rotate", "zero2", - "pair_edge", "usek", "fresh", "vpick", "three", "bump1", "bump2", "bump3", "seven"} + "pair_edge", "usek", "fresh", "vpick", "three", "bump1", "bump2", "seven", "order3", "sum3"} assert wanted <= compiled, {n: result.skipped.get(n) for n in wanted - compiled} fns = {f.name: f for f in result.module.functions} assert fns["vpick"].virtual == ("c",) and fns["fresh"].virtual == () # #143: one virtual, one real @@ -297,21 +313,48 @@ def test_no_runtime_warning_about_the_gil(extension): # --- three or more list parameters (#255) ----------------------------------------------------------- -# Since #209 a function with more than two list parameters stayed interpreted (CPython's critical -# sections cover one or two objects). #255 holds every list's own lock for the whole call instead -# (design/typedpython.md 4.3.6). Red before #255: all of the tests below. +# CPython's critical sections lock one or two objects, and the leader decision of 2026-11-13 excludes any +# other lock (design/typedpython.md 4.3.6). A function with an effect and more than two lists stays +# interpreted, as since #209. A pure one (ir.direct_lists) reads its lists element by element with +# PyList_GetItemRef, in program order, without copy-in or lock; an element of another type deopts. + +def test_pure_three_and_seven_list_functions_compile(built): + result, _, _ = built + assert {"three", "seven", "order3", "sum3"} <= {f.name for f in result.module.functions}, \ + result.skipped + -def test_three_and_seven_list_parameters_compile(built): +def test_three_list_parameters_with_an_effect_stay_interpreted(built): result, _, _ = built - assert {"three", "bump3", "seven"} <= {f.name for f in result.module.functions}, result.skipped + assert "bump3" not in {f.name for f in result.module.functions} + reason = result.skipped["bump3"] + assert "more than two list parameters" in reason, reason -def test_front_end_lowers_three_list_parameters(tmp_path): +def test_front_end_gives_a_clear_reason(tmp_path): + p = tmp_path / "m.py" + p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> None:\n" + " a[0] = 1\n") + m = frontend.lower(p) + assert not m.functions + assert "more than two list parameters" in m.skipped["f"], m.skipped["f"] + + +def test_front_end_lowers_three_list_parameters_of_a_pure_function(tmp_path): p = tmp_path / "m.py" p.write_text("\n\n@compiled\ndef f(a: list[int], b: list[float], c: list[int]) -> int:\n" " return len(a)\n") m = frontend.lower(p) - assert [f.name for f in m.functions] == ["f"], m.skipped + assert [(f.name, f.pure, f.may_deopt) for f in m.functions] == [("f", True, True)], m.skipped + + +def test_three_lists_with_an_entry_global_stay_interpreted(built): + """A module global is read once at entry; the lists are read later, one element at a time, so the + two reads could disagree with any interpreted order.""" + result, _, _ = built + assert "three_k" not in {f.name for f in result.module.functions} + reason = result.skipped["three_k"] + assert "more than two list parameters" in reason and "global" in reason, reason def test_two_list_parameters_still_compile(built): @@ -319,64 +362,63 @@ def test_two_list_parameters_still_compile(built): assert {"zero2", "pair_edge", "bump2"} <= {f.name for f in result.module.functions} -def test_two_lists_keep_the_critical_section_and_three_take_the_locks(built): - """One or two lists: CPython's own PyCriticalSection / PyCriticalSection2, unchanged. Three or - more: tp_cs_begin_n, which takes every list's lock (and can refuse, then the call deopts).""" +def _wrapper(c_text: str, name: str) -> str: + start = c_text.index(f"static PyObject *tp_wrap_{name}(") + return c_text[start:c_text.index("\n}\n", start)] + + +def test_direct_lists_take_no_lock_and_are_not_copied(built): result, path, _ = built c_text = cgen.generate(result.module, path) - def wrapper(name): - start = c_text.index(f"static PyObject *tp_wrap_{name}(") - return c_text[start:c_text.index("\n}\n", start)] - assert "tp_cs_begin(&tp_lock" in wrapper("zero2") and "tp_cs_begin_n" not in wrapper("zero2") - assert "tp_cs_begin(&tp_lock" in wrapper("fill") and "tp_cs_begin_n" not in wrapper("fill") - for name, n in (("bump3", 3), ("seven", 7)): - w = wrapper(name) - assert f"if (tp_cs_begin_n(&tp_lock, tp_lobjs, {n}) != 0) goto tp_deopt;" in w, w - assert "_exit_held(&tp_a" in w and "_array_exit(&tp_a" not in w, w + assert "tp_cs_begin(&tp_lock" in _wrapper(c_text, "zero2") # two lists: unchanged + for name in ("three", "seven", "order3"): + w = _wrapper(c_text, name) + assert "tp_cs_" not in w and "_array_enter" not in w and "_array_exit" not in w, w + assert "PyList_CheckExact" in w, w + assert "tp_list_ld_f64(l_a->list" in c_text -def test_three_lists_with_an_entry_global_stay_interpreted(built): - """Nothing between the copy-in and the write-back may wait while three or more list locks are held - (an entry global read can take the module dict's critical section).""" +def test_verifier_rejects_three_array_params(built): + from typedpython import ir result, _, _ = built - assert "three_k" not in {f.name for f in result.module.functions} - reason = result.skipped["three_k"] - assert "entry global" in reason and "three or more list" in reason, reason + f = next(f for f in result.module.functions if f.name == "zero2") + third = ir.ArrayParam("zs", ir.Type.I64_ARRAY, False) + bad = dataclasses.replace(f, params=f.params + (third,)) + mod = dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) + proved, diags = verify.verify(mod) + assert not proved.functions + assert any("array" in d.message and "two" in d.message for d in diags), diags -def _with_third_list(result, *, entry_global: bool = False): +def test_cgen_refuses_three_array_params(built): from typedpython import ir + result, path, _ = built f = next(f for f in result.module.functions if f.name == "zero2") third = ir.ArrayParam("zs", ir.Type.I64_ARRAY, False) bad = dataclasses.replace(f, params=f.params + (third,)) - if entry_global: - bad = dataclasses.replace(bad, entry_globals=(ir.Param("K", ir.Type.I64),)) - return dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) + mod = dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) + with pytest.raises(cgen.CGenError, match="two"): + cgen.generate(mod, path) -def test_verifier_accepts_three_array_params(built): - result, _, _ = built - proved, diags = verify.verify(_with_third_list(result)) - assert [f.name for f in proved.functions] == ["zero2"], diags +def _three_with_entry_global(result): + from typedpython import ir + f = next(f for f in result.module.functions if f.name == "three") + bad = dataclasses.replace(f, entry_globals=(ir.Param("K", ir.Type.I64),)) + return dataclasses.replace(result.module, functions=type(result.module.functions)([bad]), skipped={}) -def test_verifier_rejects_three_array_params_with_an_entry_global(built): +def test_verifier_rejects_direct_lists_with_an_entry_global(built): result, _, _ = built - proved, diags = verify.verify(_with_third_list(result, entry_global=True)) + proved, diags = verify.verify(_three_with_entry_global(result)) assert not proved.functions - assert any(d.rule == verify.ARRAY_LOCKS and "entry global" in d.message for d in diags), diags - - -def test_cgen_takes_three_locks(built): - result, path, _ = built - c_text = cgen.generate(_with_third_list(result), path) - assert "tp_cs_begin_n(&tp_lock, tp_lobjs, 3)" in c_text + assert any(d.rule == verify.ARRAY_DIRECT and "entry global" in d.message for d in diags), diags -def test_cgen_refuses_three_array_params_with_an_entry_global(built): +def test_cgen_refuses_direct_lists_with_an_entry_global(built): result, path, _ = built with pytest.raises(cgen.CGenError, match="entry global"): - cgen.generate(_with_third_list(result, entry_global=True), path) + cgen.generate(_three_with_entry_global(result), path) def test_three_and_seven_lists_match_the_interpreter(extension): @@ -384,36 +426,43 @@ def test_three_and_seven_lists_match_the_interpreter(extension): import random rng = random.Random(255) interp = m.__typedpython_interpreted__ + def both(name, *args): + out = [] + for fn in (getattr(m, name), interp[name]): + try: + out.append(("ok", fn(*[list(a) if type(a) is list else a for a in args]))) + except Exception as e: + out.append((type(e).__name__, str(e))) + return out before = m.__typedpython_deopts__ bad = [] - for n in (0, 1, 5, 300): + for n in (0, 1, 2, 5, 40): ls = [[rng.uniform(-3.0, 3.0) for _ in range(n)] for _ in range(7)] - a = [list(l) for l in ls] - b = [list(l) for l in ls] - ra, rb = m.seven(*a), interp["seven"](*b) - if ra != rb or a != b: - bad.append(("seven", n)) - a = [list(l) for l in ls[:3]] - b = [list(l) for l in ls[:3]] - m.bump3(*a) - interp["bump3"](*b) - if a != b: - bad.append(("bump3", n)) - ints = [list(range(n)), list(range(n + 1)), []] - if m.three(*ints) != interp["three"](*ints): - bad.append(("three", n)) + for name, args in (("seven", ls), ("three", [list(range(n)), list(range(n + 1)), []]), + ("order3", ls[:3]), ("sum3", ls[:3])): + c, i = both(name, *args) + if c != i: + bad.append((name, n, c, i)) + same_list = [1.0, 2.0] + if m.order3(same_list, same_list, same_list) != interp["order3"](same_list, same_list, same_list): + bad.append("alias") compiled_deopts = m.__typedpython_deopts__ - before - # deopts before any lock: a non-list argument and an aliased list run interpreted - x, y = [1.0, 2.0], [3.0] - m.bump3(x, x, y) - x2, y2 = [1.0, 2.0], [3.0] - interp["bump3"](x2, x2, y2) - same_alias = (x, y) == (x2, y2) - tup = m.three([1], (2, 3), [4]) - print(json.dumps({"bad": bad, "deopts": compiled_deopts, "alias": same_alias, "tuple": tup, + # deopts (then interpreted): an element of another type met by a read, a non-list argument + ls = [[1.0, 2.0, 3.0] for _ in range(7)] + ls[6] = [1.0, 2, 3.0] + for name, args in (("seven", ls), ("three", [[1], (2, 3), [4]])): + c, i = both(name, *args) + if c != i: + bad.append((name, "deopt", c, i)) + # an IndexError where the interpreter raises it (a shorter list), not a deopt + for args in ([[1.0], [], [2.0]], [[], [1.0], [2.0]]): + c, i = both("sum3", *args) + if c != i or c[0] != "IndexError": + bad.append(("sum3", "index", c, i)) + print(json.dumps({"bad": repr(bad), "deopts": compiled_deopts, "deopted": m.__typedpython_deopts__ - before - compiled_deopts})) """) - assert r == {"bad": [], "deopts": 0, "alias": True, "tuple": 4, "deopted": 2} + assert r == {"bad": "[]", "deopts": 0, "deopted": 2}, r # --- concurrent calls -------------------------------------------------------------------------------- @@ -555,129 +604,161 @@ def look(): assert r == {"bad": [], "errs": []} -# Three shared lists (#255). Every call adds 1.0 to every element of every list it is given, and each -# list is given to exactly three calls per round, so after R rounds every element is 3R exactly. A call -# whose lists are not all held from the copy-in to the write-back loses the increments another thread -# made in between (the write-back overwrites them): option (A) of #255, the mutant below. The lists stay -# below TP_POLL_INTERVAL, so no call polls (a poll writes back and copies in again, which keeps the -# count exact too, but would make the mutant's window smaller). -THREE_LISTS_BODY = """ - n = 3000 - a = [0.0] * n - b = [0.0] * n - c = [0.0] * n - R = ROUNDS - before = m.__typedpython_deopts__ - def t3(): - for _ in range(R): - m.bump3(a, b, c) - def t3r(): - for _ in range(R): - m.bump3(c, a, b) # another parameter order: the locks are taken in address order - def t1(): - for _ in range(R): - m.bump1(a) # a one-list critical section on a list held by bump3 - def t2(): - for _ in range(R): - m.bump2(c, b) # a two-list critical section (PyCriticalSection2) - errs = run_threads([t3, t3r, t1, t2]) - want = float(3 * R) - bad = [name for name, l in (("a", a), ("b", b), ("c", c)) if l != [want] * n] - lost = sum(int(want - v) for l in (a, b, c) for v in l) - print(json.dumps({"bad": bad, "lost": lost, "errs": errs, - "deopts": m.__typedpython_deopts__ - before})) +# Three shared lists read element by element (#255). The writer stores c[0] and then a[0], each one +# atomic store, with a growing value; order3 reads a[0] and then c[0]. The interpreter (and every +# compiled execution equal to an interleaving of it) sees c[0] >= a[0]: having seen the writer's a[0] +# it must see its earlier c[0]. A read out of program order (a copy of c taken before the read of a, +# as a per-list copy-in would do) can see a new a[0] with an old c[0]: the mutant below. +ORDER_BODY = """ + a = [0.0] + b = [0.0] + c = [0.0] + stop = False + N = WRITES + def writer(): + global stop + for k in range(1, N): + c[0] = float(k) + a[0] = float(k) + stop = True + bad = [] + calls = [0] + def reader(): + while True: + done = stop + d = m.order3(a, b, c) + calls[0] += 1 + if d < 0.0: + bad.append(d) + if done: + break + errs = run_threads([writer, reader, reader, reader]) + print(json.dumps({"bad": len(bad), "calls": calls[0], "errs": errs})) """ -def test_three_shared_lists_with_concurrent_writers(extension): - r = run_child(extension, THREE_LISTS_BODY.replace("ROUNDS", "(150 * STRESS)")) - assert r == {"bad": [], "lost": 0, "errs": [], "deopts": 0}, r - - -def test_seven_lists_and_a_concurrent_writer(extension): - """nbody's shape: one thread steps seven lists, another adds to one of them through a one-list - critical section; the result equals running the same calls one after the other in that order.""" - r = run_child(extension, """ - n = 2000 - ls = [[1.0] * n for _ in range(7)] - R = 100 * STRESS - before = m.__typedpython_deopts__ - def stepper(): - for _ in range(R): - m.bump3(ls[0], ls[3], ls[6]) - m.seven(*ls) - def writer(): - for _ in range(R): - m.bump1(ls[6]) - errs = run_threads([stepper, writer]) - print(json.dumps({"w": sorted(set(ls[6])), "errs": errs, "deopts": m.__typedpython_deopts__ - before})) - """) - assert r == {"w": [float(1 + 2 * 100 * STRESS)], "errs": [], "deopts": 0}, r +def test_three_lists_are_read_in_program_order(extension): + r = run_child(extension, ORDER_BODY.replace("WRITES", "(200000 * STRESS)")) + assert r["bad"] == 0 and r["errs"] == [] and r["calls"] > 0, r -def _option_a_mutant(built, tmp: Path) -> Path: - """Option (A) of #255: bump3 holds its lists' locks at the copy-in and at the write-back only (the - wrapper drops them for the impl and takes them again before the write-back).""" +def _read_order_mutant(built, tmp: Path) -> Path: + """order3 with its reads of a[0] and c[0] swapped (c first), in its impl and its fast twin.""" result, path, _ = built c_text = cgen.generate(result.module, path) - pat = re.compile(r"^( tp_s = tp_impl_bump3\(.*\);)$", re.M) - assert len(pat.findall(c_text)) == 1, "bump3's impl call not found in the generated C" - mutated = pat.sub(lambda mo: " tp_cs_end(&tp_lock);\n" + mo.group(1) - + "\n (void)tp_cs_begin_n(&tp_lock, tp_lobjs, 3);", c_text) - return cbuild.build(mutated, "ftmod", tmp / "option_a", ext_suffix=None) + ra = " tp_s = tp_list_ld_f64(l_a->list, INT64_C(0), &tp_t0);\n" + rc = " tp_s = tp_list_ld_f64(l_c->list, INT64_C(0), &tp_t2);\n" + out, pos, done = [], 0, 0 + for head in re.finditer(r"^static (?:inline )?int tp_f?impl_order3\([^\n]*\)\n\{\n", c_text, re.M): + end = c_text.index("\n}\n", head.end()) + body = c_text[head.end():end] + assert body.count(ra) == 1 and body.count(rc) == 1 and body.count(" l_x = tp_t0;\n") == 1, body + body = (body.replace(ra, "@A@").replace(rc, ra).replace("@A@", rc) # c read first, a last + .replace(" l_x = tp_t0;\n", "").replace(" l_z = tp_t2;\n", "") + .replace(" tp_t3 = l_z - l_x;\n", " l_z = tp_t2;\n l_x = tp_t0;\n tp_t3 = l_z - l_x;\n")) + assert " l_x = tp_t0;\n" in body, body + out += [c_text[pos:head.end()], body] + pos = end + done += 1 + assert done >= 1, "order3's impl not found in the generated C" + out.append(c_text[pos:]) + return cbuild.build("".join(out), "ftmod", tmp / "read_order", ext_suffix=None) @pytest.mark.race_mutant -def test_mutant_option_a_locks_at_copy_in_and_write_back_only_is_caught(built, tmp_path): - so = _option_a_mutant(built, tmp_path) - lost = 0 +def test_mutant_reading_out_of_program_order_is_caught(built, tmp_path): + so = _read_order_mutant(built, tmp_path) + caught = False for _ in range(3): - r = run_child(so, THREE_LISTS_BODY.replace("ROUNDS", "300")) - lost = r["lost"] - if lost or r["bad"]: + r = run_child(so, ORDER_BODY.replace("WRITES", "400000")) + if r["bad"]: + caught = True break - assert lost > 0, "the option (A) mutant (no lock between copy-in and write-back) was not caught" + assert caught, "the mutant (c[0] read before a[0]) was not caught" + + +def test_element_of_another_type_deopts_under_a_concurrent_writer(extension): + """b[0] flips between 1 and 1.0 while sum3 reads it: every result is 3.0 (the interpreter's, after + a deopt when the read meets the int), never a float read out of an int.""" + r = run_child(extension, """ + a, b, c = [1.0], [1], [1.0] + stop = False + def writer(): + global stop + for k in range(100000 * STRESS): + b[0] = 1 if k % 2 else 1.0 + stop = True + bad = [] + before = m.__typedpython_deopts__ + def reader(): + while True: + done = stop + v = m.sum3(a, b, c) + if v != 3.0: + bad.append(v) + if done: + break + errs = run_threads([writer, reader, reader]) + print(json.dumps({"bad": bad[:3], "errs": errs, "deopted": m.__typedpython_deopts__ > before})) + """) + assert r == {"bad": [], "errs": [], "deopted": True}, r + +def test_mutant_without_the_element_type_check_is_caught(built, tmp_path): + """The weakest point of the element-by-element read: tp_list_ld_f64's exact-float check. Without + it an int element is read as a double (deterministic, no race needed).""" + result, path, _ = built + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + check = (" if (!PyFloat_CheckExact(v)) {\n Py_DECREF(v);\n return 1;\n }\n" + " *out = PyFloat_AS_DOUBLE(v);\n") + assert h.count(check) == 1 + d = tmp_path / "rt" + d.mkdir() + (d / "tp_runtime.h").write_text(h.replace(check, " *out = PyFloat_AS_DOUBLE(v);\n")) + so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=d) + body = """ + print(json.dumps({"v": m.sum3([1.0], [1], [1.0])})) + """ + assert run_child(result.extension, body) == {"v": 3.0} + try: + caught = run_child(so, body, timeout=60)["v"] != 3.0 + except (AssertionError, subprocess.TimeoutExpired): + caught = True + assert caught, "the mutant (no exact-float check) was not caught" -def test_caller_held_critical_section_does_not_deadlock(extension): + +def test_caller_held_critical_section(extension): """list.sort holds the list's critical section while it calls the key (listobject.c, @critical_section - on list_sort_impl). A key that calls a three-list function with that list (the top-most section of - this thread) or with a list held by an outer, not top-most section must not wait for its own thread. - Results equal the interpreted function's in the same situation.""" + on list_sort_impl, and it empties the list meanwhile). A key that calls a three-list function with + that list, with a list of an outer sort, or with other lists, must not wait for its own thread, and + every call returns or raises what the interpreted function does in the same situation.""" r = run_child(extension, """ interp = m.__typedpython_interpreted__ def run(fn): - out = {} - # (1) the sorted list itself, held by the top-most section of this thread + log = [] + def call(*args): + try: + log.append(("ok", fn(*args))) + except Exception as e: + log.append((type(e).__name__, str(e))) a = [3.0, 1.0, 2.0] - b = [0.0] * 4 - c = [0.0] * 4 - a.sort(key=lambda v: (fn(a, b, c), v)[1]) - out["top"] = (a, b, c) - # (2) a list held by an outer section while an inner sort's section is the top-most + b = [5.0] * 4 + c = [7.0] * 4 + a.sort(key=lambda v: (call(a, b, c), call(b, a, c), call(b, c, b), v)[3]) outer = [2.0, 1.0] inner = [5.0, 4.0, 6.0] - b2 = [0.0] * 3 - b3 = [0.0] * 2 - c3 = [0.0] * 5 def k_inner(w): - fn(outer, b2, inner) # both sorted lists: one held by the top-most section - fn(outer, b3, c3) # only the outer one: held by a section below the top + call(outer, b, inner) + call(b, c, b) return w def k_outer(v): inner.sort(key=k_inner) + call(c, b, c) return v outer.sort(key=k_outer) - out["nested"] = (outer, inner, b2, b3, c3) - # (3) the list in the middle of the parameter list - d = [0.0] * 2 - e = [9.0, 8.0] - f = [0.0] * 2 - e.sort(key=lambda v: (fn(d, e, f), v)[1]) - out["middle"] = (d, e, f) - return out - got = run(m.bump3) - want = run(interp["bump3"]) + return log, a, b, c, outer, inner + got = run(m.order3) + want = run(interp["order3"]) print(json.dumps({"same": got == want, "got": repr(got)})) """, timeout=120) assert r["same"], r["got"] diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index 9886beff0..caaf73bc2 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -329,7 +329,8 @@ def scale(a: list[float], b: list[float], k: float) -> None: a[i] = b[i] * k """) f = function(m, "scale") - # (#255: any number of list parameters; three or more are test_free_threaded.py's) + # two list parameters at most in a function with an effect (#159: the lists are locked from + # copy-in to write-back); three or more, and pure ones (#255), are test_free_threaded.py's. assert f.params == ( ArrayParam("a", Type.F64_ARRAY, True), ArrayParam("b", Type.F64_ARRAY, False), Param("k", F64), ) From cfcd15d140356885c166acc7250e7e85ee7128c2 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:53:32 +0900 Subject: [PATCH 42/61] TypedPython: the two-list limit test covers functions with effects; pure 3-list reads are accepted (#255) --- .../src/test/python/test_buffer_arrays.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py index 724f6759e..2bab5516f 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_arrays.py @@ -509,13 +509,16 @@ def test_verifier_buffer_params_are_not_limited_but_lists_still_are(): from test_verify import F64 as VF64, F64A as VF64A, assert_accepted, assert_rejected, fn from typedpython.ir import ArrayParam, Const, Index, Return, Type - def many(n, buffer): + def many(n, buffer, pure=True, may_deopt=False): return fn("f", params=[ArrayParam(f"a{k}", VF64A, False, buffer) for k in range(n)], - returns=VF64, pure=True, + returns=VF64, pure=pure, may_deopt=may_deopt, body=[Return(Index(VF64, "a0", Const(Type.I64, 0)))]) assert_accepted(many(5, "array")) - assert_rejected("verify/structure", many(3, None)) + # #255 (d1): a pure function reads 3+ lists element by element (it may deopt on an element of + # another type); a function with effects keeps the limit of two lists + assert_accepted(many(3, None, may_deopt=True)) + assert_rejected("verify/structure", many(3, None, pure=False)) # --- vectorised element-wise store loops (#226) --------------------------------------------------- From 82fb5bf96ef652cd78a493064bb561f399c1b091 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:02:45 +0900 Subject: [PATCH 43/61] Desktop embeds free-threaded CPython 3.15t only; the free-threaded fact is per platform (#158) Maintainer decision 2026-10-04 (free-threaded 3.15t is the only build) and leader 2026-11-13 (flip the desktop default now). -PpythonFreeThreaded is removed. internal expect val embedsFreeThreadedPython: true on desktop JVM and desktop Kotlin/Native, false on Android, androidNative, iOS and wasm until #328/#175; Versions.abiFlags and Python3.autoDrainInterval read it instead of a build-wide BuildConfig flag. Published metadata per variant: the desktop jar says freeThreaded=true, the AAR false. SPEC T-1..T-4, C-4, N-1; AGENTS 12.8; ROADMAP 9. --- AGENTS.md | 7 +- README.md | 2 +- docs/SPEC.md | 53 ++++++-- docs/design/threading-and-abi.md | 37 +++--- docs/locale/README_ko.md | 2 +- docs/platforms/python-version-acquisition.md | 56 ++++++--- docs/roadmap/ROADMAP.md | 21 +++- gradle.properties | 1 - .../build.gradle.kts | 5 +- .../gradle/CPythonIncludeDirectories.kt | 28 +++-- .../gradle/EmbeddedPythonVersion.kt | 23 +++- .../gradle/PythonBindingsPlugin.kt | 6 +- .../gradle/CPythonIncludeDirectoriesTest.kt | 53 +++++--- .../gradle/EmbeddedPythonVersionTest.kt | 28 +++++ .../gradle/PythonHomeStagingTest.kt | 17 ++- .../fixtures/app/build.gradle.kts | 6 +- python-multiplatform/build.gradle.kts | 117 ++++++++++-------- .../multiplatform/AndroidGilBuildTest.kt | 34 +++++ .../EmbeddedPythonBuild.android.kt | 4 + .../multiplatform/EmbeddedPythonBuild.art.kt | 4 + .../AndroidNativeGilBuildTest.kt | 22 ++++ .../multiplatform/EmbeddedPythonBuild.kt | 30 +++++ .../kotlin/python/multiplatform/Versions.kt | 9 +- .../python/multiplatform/ffi/Python3.kt | 8 +- python-multiplatform/src/commonTest/README.md | 4 +- .../python/multiplatform/VersionsTest.kt | 12 +- .../multiplatform/ref/EvalCheckpointTest.kt | 8 +- .../EmbeddedPythonBuild.desktop.kt | 4 + .../EmbeddedPythonBuild.desktopNative.kt | 4 + .../DesktopNativeFreeThreadedBuildTest.kt | 23 ++++ .../DesktopFreeThreadedBuildTest.kt | 55 ++++++++ .../env/PublishedPythonVersionResourceTest.kt | 13 +- .../ffi/PyTypeCacheConcurrencyTest.kt | 4 +- .../multiplatform/ref/CycleCollectionTest.kt | 10 +- .../ref/FreeThreadedGCGateTest.kt | 8 +- .../ref/GCSchedulingMeasurementTest.kt | 14 ++- .../native/ffi/SidecarLibraryLookupTest.kt | 4 +- .../multiplatform/EmbeddedPythonBuild.ios.kt | 4 + .../python/multiplatform/IosGilBuildTest.kt | 22 ++++ .../EmbeddedPythonBuild.wasmJs.kt | 4 + .../python/multiplatform/WasmGilBuildTest.kt | 22 ++++ sample/build.gradle.kts | 4 +- 42 files changed, 612 insertions(+), 180 deletions(-) create mode 100644 python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt create mode 100644 python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt create mode 100644 python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt create mode 100644 python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt create mode 100644 python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt create mode 100644 python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt create mode 100644 python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt create mode 100644 python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt create mode 100644 python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt create mode 100644 python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt create mode 100644 python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt create mode 100644 python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt create mode 100644 python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt diff --git a/AGENTS.md b/AGENTS.md index 7b7791293..3b0d921b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -267,7 +267,12 @@ around one. them to force uniformity. 7. **Exposure is a blacklist.** Every `public` declaration is registered; `@PythonInternal` opts out. A whitelist (`@PythonAPI`) was rejected. -8. **Parallelism comes from free-threaded CPython (3.14t works on desktop with `-PpythonFreeThreaded=true`; default is GIL; see ROADMAP §9),** not from sub-interpreters. +8. **Parallelism comes from free-threaded CPython,** not from sub-interpreters. Maintainer, #158 + (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an + option. Baseline is CPython 3.15t." Desktop (JVM, `macosArm64`, `linuxX64`) embeds 3.15t with no + GIL option since the leader's instruction of 2026-11-13 ("flip the desktop default to 3.15t now"); + Android and iOS still embed the GIL 3.15 until #328, wasm until #175. Which build a platform embeds + is the per-platform `embedsFreeThreadedPython`, never one build-wide flag (SPEC T-1, ROADMAP §9). 9. **Do not add PanamaPort as a dependency** (licence and dependence on ART internals). ## 13. Upcalls must work in a GraalVM native image diff --git a/README.md b/README.md index f126a862d..52f756c89 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Honest summary, see the [roadmap](docs/roadmap/ROADMAP.md) for what is left to d | Binding prebuilt jars, Compose from Python | ✅ desktop · ⏳ Android / iOS / wasm | | Binding Kotlin/Native klibs | 🟡 scanner only | | `.pyi` stub generation | 🟡 partial | -| Free-threaded CPython (3.14t, desktop only, opt-in `-PpythonFreeThreaded=true`) | 🟡 partial | +| Free-threaded CPython 3.15t (the only desktop build since #158; Android / iOS still GIL until #328) | 🟡 desktop · ⏳ Android / iOS / wasm | ## 🧱 The ecosystem diff --git a/docs/SPEC.md b/docs/SPEC.md index 90d07e494..426a1519a 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -187,6 +187,9 @@ fail, then implement. re-entrant functions are never `@CriticalNative`. `Status: implemented`, `PM/desktopTest/.../ffi/JniCallConventionClassificationTest.kt`, `PM/androidInstrumentedTest/.../JniWiringTest.kt`. - **C-4** Every C API call holds the GIL. `Status: partial`, enforced by convention and README rules. + On a free-threaded build (desktop since #158, see T-2) there is no global lock, and "holds the GIL" + means the calling thread has an attached thread state (`PyGILState_Ensure`); the rule and `withGIL` + are unchanged. The GIL is released once `Python3.initialize` returns (it saves the main thread state with `PyEval_SaveThread`), so every entry into Python goes through `withGIL`, and `withoutGIL` gives it up while waiting in Kotlin. Tests: `PM/jvmTest/.../ffi/GilReleaseTest.kt` (desktop and Android @@ -643,8 +646,23 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. ## 7. Threading and builds -- **T-1** GIL builds: Kotlin threads attach a thread state and serialise on the GIL. `Status: partial` - (see C-4; the release after initialisation is tested by `GilReleaseTest` / `GilReleaseNativeTest`). +- **T-1** Which CPython build each platform embeds. Maintainer decision on #158 (2026-10-04): "Free-threaded + is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t." + Leader instruction (2026-11-13): "flip the desktop default to 3.15t now, before the #377 alpha03 + release"; mobile follows under #328, wasm under #175. So today: + desktop JVM, `macosArm64` and `linuxX64` embed the free-threaded CPython 3.15t, with no GIL option + (`-PpythonFreeThreaded` is removed and the build fails if it is passed); Android (ART and + androidNative) and iOS embed python.org's GIL CPython 3.15 and wasm a GIL CPython until those + issues land. The fact is per platform (`embedsFreeThreadedPython`, an `expect`/`actual`), and + `Versions.abiFlags` (`t` or empty) and every name derived from it (`libpython3.15t`, `lib/python3.15t/`) + follow it. On the GIL platforms Kotlin threads attach a thread state and serialise on the GIL. + `Status: partial` (desktop only; Android and iOS remain GIL until #328), + `PM/desktopTest/.../DesktopFreeThreadedBuildTest.kt` (ABI flags, `Py_GIL_DISABLED` of the loaded + interpreter, checkpoint default), `PM/desktopNativeTest/.../DesktopNativeFreeThreadedBuildTest.kt`, + `PM/androidInstrumentedTest/.../AndroidGilBuildTest.kt`, `PM/artTest/.../AndroidNativeGilBuildTest.kt`, + `PM/iosSimulatorArm64Test/.../IosGilBuildTest.kt`, `PM/wasmJsTest/.../WasmGilBuildTest.kt`, + `PM/commonTest/.../VersionsTest.kt`; the release after initialisation is tested by `GilReleaseTest` / + `GilReleaseNativeTest` (see C-4). - **T-1a** On desktop, Python may be entered from a JVM virtual thread (#161 option B, user decision): the outermost `withGIL` runs pinned to its OS carrier from `PyGILState_Ensure` to `Release`, so a virtual thread that parks inside the scope (sleep, lock, blocking I/O) stays on the OS thread that @@ -654,21 +672,32 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. Platform threads, Android, native and wasm run the block directly. A pinned virtual thread holds its carrier while it waits inside the scope, as any pinned virtual thread does. `Status: implemented`, `PM/desktopTest/.../VirtualThreadGILTest.kt` (without the pin the JVM aborts in `PyGILState_Release`). -- **T-2** Free-threaded builds selected with `-PpythonFreeThreaded=true` download a different CPython - asset. `Status: partial` (desktop only, opt-in; default is `pythonFreeThreaded=false` in `gradle.properties`) - , `GP/PythonHomeStagingTest.kt`, `PM/desktopTest/.../FreeThreadedGCGateTest.kt`, `VersionsTest.kt`. - 3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (236 tests, 0 failures; ROADMAP §9). - Only desktop has free-threaded prebuilts; Android and iOS do not. `Py_LIMITED_API` is not defined, so - `abi3t` is not a blocker. +- **T-2** The desktop build acquires python-build-standalone's free-threaded asset + (`...-freethreaded-install_only_stripped`), unpacks it under `-freethreaded`, links desktop + Kotlin/Native against `libpython3.15t`, and the Gradle plugin stages a free-threaded `PYTHONHOME` by + default (`pythonBindings.pythonFreeThreaded` defaults to `true`). `Status: partial` (desktop only, per + T-1), `GP/PythonHomeStagingTest.kt`, `PM/desktopTest/.../FreeThreadedGCGateTest.kt`, + `PM/desktopTest/.../DesktopFreeThreadedBuildTest.kt`. When it was an opt-in, 3.14t ran the whole + desktop suite (236 tests, 0 failures; ROADMAP §9); the 3.15t default has not been observed running the + suite in this revision. Android and iOS free-threaded builds are #163/#328. `Py_LIMITED_API` is not + defined, so `abi3t` is not a blocker. - **T-3** `python-multiplatform` publishes per-target, per-flavour CPython include directories as `Provider` carrying the extraction task (`cpythonIncludeDirectories` extension; see - `docs/platforms/python-version-acquisition.md` §7). `Status: partial` (only the configured flavour is - extracted), `GP/CPythonIncludeDirectoriesTest.kt`. + `docs/platforms/python-version-acquisition.md` §7). The default flavour of every accessor is the one + the library embeds for that target (`CPythonIncludeLayout.embeddedFlavour`: free-threaded on desktop, + GIL on Android and iOS; agent proposal, not confirmed). `Status: partial` (only the embedded flavour is extracted), + `GP/CPythonIncludeDirectoriesTest.kt`. - **T-4** `python-multiplatform` publishes the CPython version it embeds: extension `pythonMultiplatform`, Gradle attributes `org.thisisthepy.python.version` / `org.thisisthepy.python.free-threaded` on every consumable `*Elements` configuration, and the resource `META-INF/python-multiplatform/python.properties` in the jar and AAR (see `docs/platforms/python-version-acquisition.md` "Published version"), - `GP/EmbeddedPythonVersionTest.kt`. + `GP/EmbeddedPythonVersionTest.kt`. Per T-1 the free-threaded claim is per variant: `true` on the + desktop, `macosArm64` and `linuxX64` variants and in the jar's resource, `false` on the Android, iOS, + androidNative and wasm variants and in the AAR's resource, absent on shared metadata variants; the + extension carries the desktop value (the metadata and extension rules are agent proposals, not + confirmed). `Status: partial`, `GP/EmbeddedPythonVersionTest.kt` + (`freeThreadedIsDecidedPerVariant`), `PM/desktopTest/.../PublishedPythonVersionResourceTest.kt`; no + test reads the AAR's resource. - **T-5** Every acquired CPython reports the pinned version: `PY_VERSION` in each target's `patchlevel.h` equals `pythonVersion`, and a `downloadPython_*` task fails the build, naming the target and both versions, when it does not (issue #47: iOS shipped 3.14.6 under `pythonVersion=3.14.7`). The iOS @@ -689,7 +718,7 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. ## 9. Planned - **N-1** Sub-interpreter-free parallelism on free-threaded CPython on every platform (today desktop only, - opt-in 3.14t; see T-2 and ROADMAP §9). `Status: partial`. + 3.15t by default since #158; Android and iOS under #328; see T-1, T-2 and ROADMAP §9). `Status: partial`. - **N-2** Compose through Python on Android, iOS and wasm. `Status: planned`. - **N-3** Binding Kotlin/Native klib declarations at run time (B-2). `Status: planned`. - **N-4** Linux and Windows desktop runs in CI (P-2). `Status: planned`. diff --git a/docs/design/threading-and-abi.md b/docs/design/threading-and-abi.md index 22865f0cc..71f37691c 100644 --- a/docs/design/threading-and-abi.md +++ b/docs/design/threading-and-abi.md @@ -5,17 +5,22 @@ Parallelism comes from **free-threaded CPython**, not from per-interpreter GIL sub-interpreters (AGENTS.md §12.8). -The library supports **two flavours**, chosen at build time: +Maintainer, #158 (2026-10-04): "Free-threaded is the default and the only build. The GIL build is +not kept as an option. Baseline is CPython 3.15t." Desktop moved first (leader, 2026-11-13: "flip the +desktop default to 3.15t now, before the #377 alpha03 release"); the other platforms follow, so today +the build differs by platform (SPEC T-1): -| Interpreter | Threading behaviour | Status | +| Platform | Interpreter | Threading behaviour | |---|---|---| -| `python3.X` (GIL) | Kotlin threads each attach a thread state and serialise on the GIL | default (`pythonFreeThreaded=false` in `gradle.properties`) | -| `python3.Xt` (free-threaded) | Each Kotlin thread talks to its own free-running Python thread | **3.14t works on desktop**, opt-in with `-PpythonFreeThreaded=true` | - -The interpreter version is pinned by `pythonVersion` in `gradle.properties` (3.14.7 at the time of -writing). Free-threading is **not** gated on 3.15t: the whole desktop suite runs on 3.14.7t -(236 tests, 0 failures, 1 skipped, ROADMAP §9; SPEC T-2). Only desktop has free-threaded -prebuilts; Android and iOS have none, so the opt-in is desktop-only. How the earlier "from 3.15t" +| Desktop JVM, `macosArm64`, `linuxX64` | `python3.15t` (free-threaded), the only desktop build | Each Kotlin thread talks to its own free-running Python thread | +| Android (ART, androidNative), iOS | `python3.15` (GIL) until #328 | Kotlin threads each attach a thread state and serialise on the GIL | +| wasmJs | GIL until #175 | single-threaded | + +The per-platform fact is `embedsFreeThreadedPython` (commonMain `EmbeddedPythonBuild.kt`, an +`expect`/`actual`); `-PpythonFreeThreaded` is removed. The interpreter version is pinned by +`pythonVersion` in `gradle.properties` (3.15.0). Before #158 free-threading was a desktop opt-in, and +the whole desktop suite ran on 3.14.7t (236 tests, 0 failures, 1 skipped, ROADMAP §9; SPEC T-2); the +3.15t default has not yet been observed running the suite. How the earlier "from 3.15t" conclusion was reached, and why it did not hold, is in [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). @@ -93,13 +98,13 @@ Design for two artefacts, but keep the split shallow enough to collapse later. ## Prebuilt availability Desktop (macOS arm64/x86_64, Linux x86_64, Windows x86_64) has GIL and free-threaded builds on the -python-build-standalone releases; the build uses the `-freethreaded` archive when -`pythonFreeThreaded=true`. **Android (python.org) and iOS (Python-Apple-support) publish no -free-threaded build**, so free-threading stays a desktop-only opt-in, and enabling it by default -would split the threading model, and the object-lifetime and thread-state design layered on top, -across platforms. Switching the default would additionally need adequate free-threaded wheel -coverage for the C extensions users care about. Table as of the 20260807 release, kept in the -archive note above. +python-build-standalone releases; the build uses only the `-freethreaded` archive (#158). +**Android (python.org) and iOS (Python-Apple-support) publish no free-threaded build**, so those +platforms stay GIL until the project's own builds land (#163, #328). Until then the threading model, +and the object-lifetime and thread-state design layered on top, are split across platforms; that is +the accepted cost of moving desktop first. The maintainer accepted the wheel-coverage cost on #158 +("third-party cp315t / abi3t wheels lag the release for a while; pypackpack builds from sdist where +they are missing"). Table as of the 20260807 release, kept in the archive note above. ## Thread-state management diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md index fe74fad20..b53e50ba1 100644 --- a/docs/locale/README_ko.md +++ b/docs/locale/README_ko.md @@ -196,7 +196,7 @@ Python → Kotlin 바인딩 코드가 없다. 자세한 내용: [설계 문서]( | 빌드된 jar 바인딩, Python 에서 Compose | ✅ 데스크톱 · ⏳ Android / iOS / wasm | | Kotlin/Native klib 바인딩 | 🟡 스캐너만 | | `.pyi` 스텁 생성 | 🟡 부분 | -| free-threaded CPython (3.14t, desktop 한정, `-PpythonFreeThreaded=true` 옵트인) | 🟡 부분 | +| free-threaded CPython 3.15t (#158 이후 데스크톱의 유일한 빌드; Android / iOS 는 #328 까지 GIL) | 🟡 데스크톱 · ⏳ Android / iOS / wasm | ## 🧱 생태계 diff --git a/docs/platforms/python-version-acquisition.md b/docs/platforms/python-version-acquisition.md index 544c13d3f..557dd7fcb 100644 --- a/docs/platforms/python-version-acquisition.md +++ b/docs/platforms/python-version-acquisition.md @@ -23,6 +23,12 @@ The project currently uses `python-build-standalone` (maintained by Astral). For * **Naming:** The string `freethreaded` is injected into the asset name before the flavour, e.g., `cpython-3.14.7+20260807-aarch64-apple-darwin-freethreaded-install_only.tar.gz`. * **ABI and Libraries:** The ABI suffix is `t`, meaning the shared libraries will be named `libpython3.14t.dylib` or `libpython3.14t.so`. * **Availability:** They are available for all major desktop targets (macOS, Linux, Windows). +* **Acquired: the free-threaded build only, on desktop** (issue #158; maintainer 2026-10-04: "Free-threaded + is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t"; + leader 2026-11-13: "flip the desktop default to 3.15t now"). Every desktop `downloadPython_*` task fetches + `...-freethreaded-install_only_stripped.tar.gz` and unpacks it under `/-freethreaded/`. + The `-PpythonFreeThreaded` property that used to choose between the two is removed; passing it fails the + build. Android and iOS (section 3) are still GIL until #328. ## 3. Android and iOS Support @@ -38,15 +44,17 @@ The project currently uses `python-build-standalone` (maintained by Astral). For * **The constraint:** the ~330 bindings target the CPython Stable ABI, and CPython's Limited API / `abi3` is not offered by free-threaded builds in 3.13 and 3.14 (3.15 adds `abi3t`, PEP 803). * **Why it does not block free-threading here:** the bindings are symbols resolved dynamically at run - time and `Py_LIMITED_API` is not defined, so no compile-time ABI contract applies. 3.14t works on - desktop with `-PpythonFreeThreaded=true` (the whole desktop suite: 236 tests, 0 failures, at the time - of ROADMAP §9; SPEC T-2). An earlier version of this section concluded the opposite ("fails or crashes + time and `Py_LIMITED_API` is not defined, so no compile-time ABI contract applies. 3.14t worked on + desktop when it was an opt-in (the whole desktop suite: 236 tests, 0 failures, at the time of ROADMAP + §9; SPEC T-2); desktop now embeds 3.15t only (section 2). An earlier version of this section concluded the opposite ("fails or crashes due to opaque structural changes", "wait for 3.15t") and was wrong; see [`../design/threading-and-abi.md`](../design/threading-and-abi.md) and [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). * **Scope:** free-threaded prebuilts exist only for the desktop targets (section 2); Android (python.org) - and iOS (Python-Apple-support) publish none, so the flag is desktop-only and defaults to `false` - (`gradle.properties`). The interpreter version is `pythonVersion` in `gradle.properties`. + and iOS (Python-Apple-support) publish none, so desktop is free-threaded and Android and iOS stay GIL + until the project's own builds land (#163, #328). Which build each platform embeds is the library's + per-platform `embedsFreeThreadedPython` (SPEC T-1). The interpreter version is `pythonVersion` in + `gradle.properties`. ## 5. Security and Integrity Verification @@ -76,24 +84,26 @@ Compiling a C extension against the embedded CPython needs that CPython's header * **Key:** `(target, flavour)`. Targets: `macos-aarch64`, `macos-x86_64`, `linux-x86_64`, `windows-x86_64`, `android-aarch64`, `android-x86_64`, `ios-arm64`, `ios-arm64_x86_64-simulator` - (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.GIL` (default) or `FREE_THREADED` - (desktop only). + (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.FREE_THREADED` or `GIL`; the default + is the one the library embeds for that target, `CPythonIncludeLayout.embeddedFlavour(target)`: + `FREE_THREADED` on desktop (#158), `GIL` on Android and iOS. (Making the default per target, rather + than `GIL` everywhere, is an agent proposal, not confirmed: with `GIL` the documented + `includeDir("macos-aarch64")` would throw on every desktop build.) * **Value:** `Provider` of the directory that directly contains `Python.h`, i.e. the `-I` directory: `/[-freethreaded]/python/include/python3.14[t]` on desktop (Windows has no `python3.14` level), `/android-*/prefix/include/python3.14`, `/ios/Python.xcframework//include/python3.14`. * **Task dependency:** the provider carries the matching `downloadPython_*` task; using it as a task input makes Gradle download and extract first. -* **Only the configured flavour is extracted.** `-PpythonFreeThreaded=true` selects the free-threaded - desktop tree; asking for the other flavour throws a `GradleException` naming that property - (`isAvailable` tests it without throwing). +* **Only the embedded flavour is extracted.** Desktop extracts the free-threaded tree only; asking for the + other flavour throws a `GradleException` naming #158 (`isAvailable` tests it without throwing). ```kotlin // consumer build.gradle.kts evaluationDependsOn(":python-multiplatform") val includes = project(":python-multiplatform").extensions.getByType() tasks.register("compileExt") { - val inc = includes.includeDir("macos-aarch64") // or ("linux-x86_64", CPythonFlavour.FREE_THREADED) + val inc = includes.includeDir("macos-aarch64") // the free-threaded tree, .../include/python3.15t inputs.dir(inc) commandLine("cc", "-I${inc.get().asFile}", "-c", "ext.c") } @@ -112,22 +122,30 @@ The same extension exposes the library to link against (issue #56): | `windows-x86_64` free-threaded | `windows-x86_64-freethreaded/python/libs/python314t.lib` | | macOS, Linux, iOS | none -- `isLinkRequired(target)` is false; `libraryDir`/`libraryFile` throw `GradleException` | -Android has no free-threaded build, so that combination throws. `isLinkAvailable(target, flavour)` -tests without throwing; the other flavour fails naming `-PpythonFreeThreaded`, as for `includeDir`. +Android has no free-threaded build, so that combination throws. On Windows the default is the +free-threaded `python315t.lib`; the GIL one is not extracted. `isLinkAvailable(target, flavour)` +tests without throwing; the other flavour fails naming #158, as for `includeDir`. ## Published version A consumer that builds against `python-multiplatform` must know which CPython it embeds (issue #61). -The answer is published three ways, all derived from `pythonVersion` / `pythonFreeThreaded` in -`gradle.properties`. +The answer is published three ways. The version is `pythonVersion` in `gradle.properties`; whether +the build is free-threaded differs by platform (#158: desktop 3.15t, Android and iOS GIL 3.15 until +#328, wasm GIL until #175), so each published artefact states its own. * **Extension** `pythonMultiplatform` (`python.multiplatform.gradle.EmbeddedPythonVersion`): - `pythonVersion` (`3.14.7`), `majorMinor` (`3.14`), `freeThreaded`. + `pythonVersion` (`3.15.0`), `majorMinor` (`3.15`), `freeThreaded`. One value per project: the + desktop one (`true`; agent proposal, not confirmed). For any other platform, read its variant's + attribute or its resource. * **Gradle attributes** on every consumable `*Elements` configuration (so they land in the published module metadata): `org.thisisthepy.python.version` and `org.thisisthepy.python.free-threaded` - (both `String`; the latter is `"true"`/`"false"`). + (both `String`; the latter is `"true"`/`"false"`). The free-threaded one is per variant: `"true"` on + `desktop*`, `macosArm64*` and `linuxX64*` variants, `"false"` on the Android, iOS, androidNative and + wasm variants, and absent on shared metadata variants, which carry no CPython of their own (the + metadata rule is an agent proposal, not confirmed). * **Resource** `META-INF/python-multiplatform/python.properties` (`pythonVersion=`, `freeThreaded=`), - inside the desktop jar and the Android AAR's `classes.jar`. + inside the desktop jar (`freeThreaded=true`, from `generateDesktopPythonProperties`) and the Android + AAR's `classes.jar` (`freeThreaded=false`, from `generateAndroidPythonProperties`). ```kotlin // composite build (includeBuild): read the extension @@ -142,6 +160,6 @@ val embedded = EmbeddedPythonVersion.parse(text) // published artefact: require a matching variant through the attributes configurations.named("desktopRuntimeClasspath") { - attributes.attribute(Attribute.of("org.thisisthepy.python.version", String::class.java), "3.14.7") + attributes.attribute(Attribute.of("org.thisisthepy.python.version", String::class.java), "3.15.0") } ``` diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index 029df0d52..547a8f162 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -160,9 +160,14 @@ JNI callee names). Re-opening it means redesigning Android's convention selectio ## 9. Free-threading -3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (opt-in; `gradle.properties` -default `pythonFreeThreaded=false`): 236 tests, 0 failures, 1 skipped when last recorded, the same -count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is +Desktop (JVM, `macosArm64`, `linuxX64`) embeds the free-threaded CPython 3.15t and only that build +(maintainer, #158, 2026-10-04: "Free-threaded is the default and the only build. The GIL build is not +kept as an option. Baseline is CPython 3.15t"; leader, 2026-11-13: "flip the desktop default to 3.15t +now, before the #377 alpha03 release"). `-PpythonFreeThreaded` is removed; the build fails if it is +passed. Android and iOS still embed the GIL 3.15 (9.2), wasm GIL (#175); the per-platform fact is +`embedsFreeThreadedPython` (SPEC T-1). The 3.15t default has not yet been observed running the desktop +suite; the last recorded free-threaded run is 3.14t as an opt-in: 236 tests, 0 failures, 1 skipped, the +same count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is drained by the eval-loop checkpoint (`PM/commonTest/.../ref/EvalCheckpointTest.kt`, `docs/investigations/gc-scheduling-investigation.md`); `FreeThreadedGCGateTest` guards the GC probe. The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done @@ -170,15 +175,19 @@ The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done free-threaded ones, are pinned in `python-checksums.properties`. - **9.1 `autoDrainInterval`'s default on the GIL build, open (decision).** Today - `if (BuildConfig.pythonFreeThreaded) 32 else 0` (`Python3.kt`). Off means a pure C-API embedder never + `if (embedsFreeThreadedPython) 32 else 0` (`Python3.kt`), so on by default on desktop and off on + Android and iOS until they move (9.2); once every platform is free-threaded this question closes. Off means a pure C-API embedder never runs the cyclic collector: 20 000+ cyclic objects left with the automatic path off against ~1 970 with it on (`docs/investigations/gc-scheduling-investigation.md` §6–§7). On means a `__del__` or weakref callback can run at a `withGIL` exit the caller did not ask to yield from. The checkpoint costs ~128 ns over a 154.6 ns `withGIL` floor, ~4 ns amortised (`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`). A one-line change once decided. -- **9.2 Free-threading beyond desktop, open (blocked upstream).** No free-threaded CPython prebuilt +- **9.2 Free-threading beyond desktop, open (#328, wasm #175).** No free-threaded CPython prebuilt exists for Android (python.org ships `libpython3.14.so`/`3.15.so` only) or iOS (neither BeeWare - nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12. SPEC N-1. + nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12; the project builds its own (#163, + `.github/workflows/build-ft-mobile.yml`). Moving a platform means flipping its + `embedsFreeThreadedPython` actual together with its acquisition, its `python.properties` resource + and its variant attributes. SPEC N-1. ## 10. WASM diff --git a/gradle.properties b/gradle.properties index 069e2f372..c800b0ec5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -28,7 +28,6 @@ org.jetbrains.compose.hot.reload.disable=true # Python Build pythonVersion=3.15.0 -pythonFreeThreaded=false pythonBuildStandaloneRelease=20261013 pythonAppleSupportBuild=b11 # Every published artifact (library, KSP processor, Gradle plugin) is versioned diff --git a/python-multiplatform-gradle-plugin/build.gradle.kts b/python-multiplatform-gradle-plugin/build.gradle.kts index 5c20b0b6f..e91ee55c3 100644 --- a/python-multiplatform-gradle-plugin/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/build.gradle.kts @@ -251,7 +251,10 @@ val generateCoordinates = tasks.register("generateCoordinates") { // case where this build is checked out without the root's properties beside it. val pythonVersionValue = rootProperty("pythonVersion", "3.15.0") val pbsReleaseValue = rootProperty("pythonBuildStandaloneRelease", "20261013") - val freeThreadedValue = rootProperty("pythonFreeThreaded", "false").toBoolean() + // Desktop embeds the free-threaded build only (issue #158, maintainer 2026-10-04: "Free-threaded + // is the default and the only build. The GIL build is not kept as an option."), so there is no + // root property to read any more: the library's `desktopJar` always carries `libpython3.15t`. + val freeThreadedValue = true // The library and this plugin publish under one version since #70, so the wasm-runtime zip // (published beside the library artifact) is at this build's own version. diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt index 5b4862399..f5c87597e 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt @@ -28,6 +28,14 @@ object CPythonIncludeLayout { val allTargets: List = desktopTargets + androidTargets + iosTargets + /** + * The flavour `python-multiplatform` embeds for [target], which is the only one its build + * extracts: free-threaded on desktop (issue #158, maintainer 2026-10-04: "Free-threaded is the + * default and the only build"), GIL on Android and iOS until #328. + */ + fun embeddedFlavour(target: String): CPythonFlavour = + if (target in desktopTargets) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL + fun supports(target: String, flavour: CPythonFlavour): Boolean = target in allTargets && (flavour == CPythonFlavour.GIL || target in desktopTargets) @@ -97,9 +105,9 @@ object CPythonIncludeLayout { * task as its build dependency, so anything that wires it as an input (`-I`, an `@InputDirectory`) * triggers the download and unpack. * - * Only the flavour the build was configured for (`-PpythonFreeThreaded`) has an extraction task; - * asking for the other one fails with a message naming the property, rather than returning a path - * nothing will ever create. + * Only the flavour the library embeds ([CPythonIncludeLayout.embeddedFlavour]: free-threaded on + * desktop, GIL on Android and iOS) has an extraction task, and it is the default `flavour`; asking + * for the other one fails loudly rather than returning a path nothing will ever create. * * @param extractedRoot the version-keyed extraction directory * @param extractionTask the task that fills [extractedRoot] for a target+flavour, or null when this build does not extract it @@ -112,11 +120,11 @@ class CPythonIncludeDirectories( val targets: List get() = CPythonIncludeLayout.allTargets /** Whether [includeDir] would succeed for this pair in this build. */ - fun isAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean = + fun isAvailable(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Boolean = CPythonIncludeLayout.supports(target, flavour) && extractionTask(target, flavour) != null /** The directory holding `Python.h` for [target]; resolving it makes Gradle run the extraction task first. */ - fun includeDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + fun includeDir(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { if (!CPythonIncludeLayout.supports(target, flavour)) { throw GradleException( "No $flavour CPython include directory for '$target'. Targets: ${CPythonIncludeLayout.allTargets}; " + @@ -125,7 +133,7 @@ class CPythonIncludeDirectories( } val task = extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour selected with -PpythonFreeThreaded is downloaded.", + "Only the flavour the library embeds is downloaded: free-threaded on desktop (#158), GIL on Android and iOS.", ) val relative = CPythonIncludeLayout.relativePath(target, flavour, pythonVersion) // `TaskProvider.map` keeps the task as the producer, so the dependency travels with the value. @@ -136,7 +144,7 @@ class CPythonIncludeDirectories( fun isLinkRequired(target: String): Boolean = CPythonIncludeLayout.isLinkRequired(target) /** Whether [libraryDir]/[libraryFile] would succeed for this pair in this build. */ - fun isLinkAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean = + fun isLinkAvailable(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Boolean = CPythonIncludeLayout.supportsLinkLibrary(target, flavour) && extractionTask(target, flavour) != null private fun linkTask(target: String, flavour: CPythonFlavour): TaskProvider<*> { @@ -148,19 +156,19 @@ class CPythonIncludeDirectories( } return extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour selected with -PpythonFreeThreaded is downloaded.", + "Only the flavour the library embeds is downloaded: free-threaded on desktop (#158), GIL on Android and iOS.", ) } /** The directory holding the link library (the `-L` directory); resolving it runs the extraction task first. */ - fun libraryDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + fun libraryDir(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { val task = linkTask(target, flavour) val relative = CPythonIncludeLayout.linkLibraryDirRelativePath(target, flavour) return task.map { extractedRoot.dir(relative) } } /** The link library file itself (`libpython3.14.so` / `python314.lib`); carries the extraction task. */ - fun libraryFile(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + fun libraryFile(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { val task = linkTask(target, flavour) val relative = CPythonIncludeLayout.linkLibraryDirRelativePath(target, flavour) + "/" + CPythonIncludeLayout.linkLibraryFileName(target, flavour, pythonVersion) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt index 2b4a714fe..d1bf42786 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt @@ -10,6 +10,11 @@ import org.gradle.api.Project * - as the Gradle attributes [ATTRIBUTE_VERSION] / [ATTRIBUTE_FREE_THREADED] on every consumable * `*Elements` configuration (published Gradle module metadata), * - as the classpath resource [RESOURCE_PATH] (read from the jar/AAR, no Gradle needed). + * + * One instance describes one build of CPython. A project whose platforms embed different builds + * (issue #158: desktop 3.15t, Android and iOS the GIL 3.15 until #328) passes a per-variant + * `freeThreadedFor` to [publishAttributes] / [register], so each published variant states the build + * it really carries, and renders one resource per artefact. */ class EmbeddedPythonVersion(val pythonVersion: String, val freeThreaded: Boolean) { @@ -23,22 +28,30 @@ class EmbeddedPythonVersion(val pythonVersion: String, val freeThreaded: Boolean fun renderProperties(): String = "pythonVersion=$pythonVersion\nfreeThreaded=$freeThreaded\n" - /** Adds the two attributes to every consumable `*Elements` configuration, including ones created later. */ - fun publishAttributes(project: Project) { + /** + * Adds the attributes to every consumable `*Elements` configuration, including ones created later. + * + * [ATTRIBUTE_VERSION] is [pythonVersion] everywhere. [ATTRIBUTE_FREE_THREADED] is + * `freeThreadedFor(configurationName)`: by default [freeThreaded] for every variant; a + * multi-platform project passes a function that answers per variant, and `null` leaves the + * attribute off a variant that carries no CPython of its own (shared metadata). + */ + fun publishAttributes(project: Project, freeThreadedFor: (configurationName: String) -> Boolean? = { freeThreaded }) { project.configurations .matching { it.isCanBeConsumed && it.name.endsWith("Elements") } .configureEach { attributes.attribute(org.gradle.api.attributes.Attribute.of(ATTRIBUTE_VERSION, String::class.java), pythonVersion) + val ft = freeThreadedFor(name) ?: return@configureEach attributes.attribute( org.gradle.api.attributes.Attribute.of(ATTRIBUTE_FREE_THREADED, String::class.java), - freeThreaded.toString(), + ft.toString(), ) } } - fun register(project: Project) { + fun register(project: Project, freeThreadedFor: (configurationName: String) -> Boolean? = { freeThreaded }) { project.extensions.add(EXTENSION_NAME, this) - publishAttributes(project) + publishAttributes(project, freeThreadedFor) } companion object { diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index 895f42e4e..0eb32b7bd 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -159,7 +159,11 @@ interface PythonBindingsExtension { /** python-build-standalone release tag to stage from. Defaults to the one this library used. */ val pythonBuildStandaloneRelease: Property - /** Whether to stage a free-threaded build. Defaults to what this library was built against. */ + /** + * Whether to stage a free-threaded build. Defaults to `true`, what the desktop library embeds + * (issue #158: desktop is free-threaded CPython 3.15t only). `false` stages a GIL prefix whose + * `lib/python3.15/` the desktop jar's `libpython3.15t` does not look for. + */ val pythonFreeThreaded: Property /** diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt index b7b199545..156c2051c 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt @@ -80,7 +80,8 @@ class CPythonIncludeDirectoriesTest { @Test fun everyTargetResolvesToADistinctDirectoryHoldingPythonH() { - val (project, dirs) = fixture(CPythonFlavour.GIL) + // FREE_THREADED is the desktop flavour the library build extracts (#158). + val (project, dirs) = fixture(CPythonFlavour.FREE_THREADED) val seen = mutableSetOf() for (t in dirs.targets) { val producer = project.tasks.getByName("fake_${t.replace('-', '_')}") @@ -99,14 +100,37 @@ class CPythonIncludeDirectoriesTest { } @Test - fun theOtherFlavourFailsLoudlyNamingTheProperty() { - val (_, dirs) = fixture(CPythonFlavour.GIL) - assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.FREE_THREADED)) - val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.FREE_THREADED) } - assertTrue("pythonFreeThreaded" in e.message.orEmpty()) + fun theOtherFlavourFailsLoudlyNamingTheDecision() { + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) + assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.GIL)) + val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.GIL) } + assertTrue("#158" in e.message.orEmpty(), e.message) assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.FREE_THREADED) } } + /** + * Issue #158: desktop embeds only the free-threaded build, so the default flavour of every + * accessor is the one the library embeds for that target. Red before: the default was + * `CPythonFlavour.GIL` for every target, so `includeDir("macos-aarch64")` asked for a tree the + * free-threaded build does not extract and threw (and `embeddedFlavour` did not exist, so this + * does not compile against the old source). + */ + @Test + fun theDefaultFlavourIsTheOneTheLibraryEmbeds() { + for (t in CPythonIncludeLayout.desktopTargets) { + assertEquals(CPythonFlavour.FREE_THREADED, CPythonIncludeLayout.embeddedFlavour(t), t) + } + for (t in CPythonIncludeLayout.androidTargets + CPythonIncludeLayout.iosTargets) { + assertEquals(CPythonFlavour.GIL, CPythonIncludeLayout.embeddedFlavour(t), t) + } + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) + assertTrue(dirs.isAvailable("macos-aarch64")) + val root = dirs.includeDir("macos-aarch64").get().asFile.parentFile.parentFile.parentFile.parentFile + assertEquals(File(root, "macos-aarch64-freethreaded/python/include/python3.14t"), dirs.includeDir("macos-aarch64").get().asFile) + assertEquals(File(root, "android-aarch64/prefix/include/python3.14"), dirs.includeDir("android-aarch64").get().asFile) + assertEquals("python314t.lib", dirs.libraryFile("windows-x86_64").get().asFile.name) + } + @Test fun configuredFreeThreadedExposesFreeThreadedKey() { val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) @@ -148,7 +172,7 @@ class CPythonIncludeDirectoriesTest { @Test fun linkLibraryProvidersCarryTheExtractionTaskAndResolve() { - val (project, dirs) = fixture(CPythonFlavour.GIL) + val (project, dirs) = fixture(CPythonFlavour.FREE_THREADED) val root = project.layout.projectDirectory.dir("extracted/$version") for (t in dirs.targets.filter { dirs.isLinkRequired(it) }) { val producer = project.tasks.getByName("fake_${t.replace('-', '_')}") @@ -159,23 +183,24 @@ class CPythonIncludeDirectoriesTest { val fileConsumer = project.tasks.register("linkf_${t.replace('-', '_')}") { inputs.file(fileProvider) }.get() assertTrue(producer in fileConsumer.taskDependencies.getDependencies(fileConsumer), "$t: libraryFile lacks task") // Simulate the extraction creating the library. - val expected = root.dir(CPythonIncludeLayout.linkLibraryDirRelativePath(t, CPythonFlavour.GIL)).asFile + val flavour = CPythonIncludeLayout.embeddedFlavour(t) + val expected = root.dir(CPythonIncludeLayout.linkLibraryDirRelativePath(t, flavour)).asFile expected.mkdirs() - File(expected, CPythonIncludeLayout.linkLibraryFileName(t, CPythonFlavour.GIL, version)).writeText("lib") + File(expected, CPythonIncludeLayout.linkLibraryFileName(t, flavour, version)).writeText("lib") assertEquals(expected, dirProvider.get().asFile) - assertEquals(expected.resolve(CPythonIncludeLayout.linkLibraryFileName(t, CPythonFlavour.GIL, version)), fileProvider.get().asFile) + assertEquals(expected.resolve(CPythonIncludeLayout.linkLibraryFileName(t, flavour, version)), fileProvider.get().asFile) } } @Test fun linkLibraryFailsLoudlyWhenNotRequiredOrWrongFlavour() { - val (_, dirs) = fixture(CPythonFlavour.GIL) + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) assertFalse(dirs.isLinkRequired("macos-aarch64")) assertFailsWith { dirs.libraryDir("macos-aarch64") } assertFailsWith { dirs.libraryFile("ios-arm64") } - assertFalse(dirs.isLinkAvailable("windows-x86_64", CPythonFlavour.FREE_THREADED)) - val e = assertFailsWith { dirs.libraryDir("windows-x86_64", CPythonFlavour.FREE_THREADED) } - assertTrue("pythonFreeThreaded" in e.message.orEmpty()) + assertFalse(dirs.isLinkAvailable("windows-x86_64", CPythonFlavour.GIL)) + val e = assertFailsWith { dirs.libraryDir("windows-x86_64", CPythonFlavour.GIL) } + assertTrue("#158" in e.message.orEmpty(), e.message) assertFailsWith { dirs.libraryDir("android-aarch64", CPythonFlavour.FREE_THREADED) } } diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt index 09d1c5512..442b6d1d2 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/EmbeddedPythonVersionTest.kt @@ -52,6 +52,34 @@ class EmbeddedPythonVersionTest { } } + /** + * Issue #158: desktop embeds 3.15t while Android and iOS embed the GIL 3.15 until #328, so the + * free-threaded attribute is decided per variant. Red before: `register` took no per-variant + * function and stamped one value on every variant, so the AAR's variants claimed what the + * desktop jar's did (this test then does not compile, which is the pre-implementation state). + */ + @Test + fun freeThreadedIsDecidedPerVariant() { + val project = ProjectBuilder.builder().build() + val names = listOf("desktopRuntimeElements", "releaseRuntimeElements", "iosArm64ApiElements", "metadataApiElements") + val configs = names.associateWith { n -> project.configurations.create(n) { isCanBeConsumed = true } } + EmbeddedPythonVersion("3.15.0", true).register(project) { name -> + when { + name.startsWith("metadata") -> null + name.startsWith("desktop") -> true + else -> false + } + } + val ft = attr(EmbeddedPythonVersion.ATTRIBUTE_FREE_THREADED) + assertEquals("true", configs.getValue("desktopRuntimeElements").attributes.getAttribute(ft)) + assertEquals("false", configs.getValue("releaseRuntimeElements").attributes.getAttribute(ft)) + assertEquals("false", configs.getValue("iosArm64ApiElements").attributes.getAttribute(ft)) + assertNull(configs.getValue("metadataApiElements").attributes.getAttribute(ft)) + for (c in configs.values) { + assertEquals("3.15.0", c.attributes.getAttribute(attr(EmbeddedPythonVersion.ATTRIBUTE_VERSION))) + } + } + @Test fun nonConsumableConfigurationsAreLeftAlone() { val project = ProjectBuilder.builder().build() diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt index 839ba3751..9e7ab6b92 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt @@ -75,9 +75,24 @@ class PythonHomeStagingTest { ) } + /** + * Issue #158: the desktop library embeds only the free-threaded CPython 3.15t, so the prefix a + * consumer stages by default must be the free-threaded one. Red before: the default was read + * from the root's `pythonFreeThreaded=false`, and the staged `lib/python3.15/` was a directory + * the desktop jar's `libpython3.15t` does not look in. + */ + @Test + fun theStagedPrefixDefaultsToTheFreeThreadedBuild() { + assertTrue(DEFAULT_PYTHON_FREE_THREADED, "desktop is free-threaded only (#158)") + assertEquals( + "cpython-$DEFAULT_PYTHON_VERSION+$DEFAULT_PBS_RELEASE-aarch64-apple-darwin-freethreaded-install_only_stripped.tar.gz", + pbsAssetName(DEFAULT_PYTHON_VERSION, DEFAULT_PBS_RELEASE, "macos-aarch64", DEFAULT_PYTHON_FREE_THREADED), + ) + } + @Test fun aFreeThreadedBuildIsADifferentAssetNotAFlagOnTheSameOne() { - // `-PpythonFreeThreaded=true` selects a different tarball upstream, and the resulting + // The free-threaded build is a different tarball upstream, and the resulting // prefix holds `lib/python3.14t/` rather than `lib/python3.14/`. Staging the default // tarball for a free-threaded consumer produces a prefix `PythonHomeCheck` rejects. assertEquals( diff --git a/python-multiplatform-ksp/fixtures/app/build.gradle.kts b/python-multiplatform-ksp/fixtures/app/build.gradle.kts index 9cd4ac650..53bd8351a 100644 --- a/python-multiplatform-ksp/fixtures/app/build.gradle.kts +++ b/python-multiplatform-ksp/fixtures/app/build.gradle.kts @@ -68,7 +68,7 @@ pythonBindings { // // - headers: `python-multiplatform`'s `cpythonIncludeDirectories` provider // (`docs/platforms/python-version-acquisition.md` §7), for this host's macOS target and the -// flavour `-PpythonFreeThreaded` selects. The provider carries the `downloadPython_*` task. +// free-threaded flavour, the only one desktop embeds (#158). The provider carries the `downloadPython_*` task. // - extension suffix: `typedpython/build_bridge.py` reads `EXT_SUFFIX` from the `_sysconfigdata_*.py` // of that same extracted tree and cross-checks `Py_GIL_DISABLED` against its `pyconfig.h`. // @@ -123,8 +123,8 @@ tasks.matching { it.name == "emitTypedPythonC" }.configureEach { val typedPythonIncludeDir: Provider? = typedPythonMacTarget?.let { target -> val includes = project(":python-multiplatform").extensions.getByName("cpythonIncludeDirectories") val flavourClass = includes.javaClass.classLoader.loadClass("python.multiplatform.gradle.CPythonFlavour") - val freeThreaded = (findProperty("pythonFreeThreaded") as String?)?.toBoolean() ?: false - val flavour = flavourClass.enumConstants.single { (it as Enum<*>).name == if (freeThreaded) "FREE_THREADED" else "GIL" } + // Desktop embeds only the free-threaded build (#158). + val flavour = flavourClass.enumConstants.single { (it as Enum<*>).name == "FREE_THREADED" } includes.javaClass.getMethod("includeDir", String::class.java, flavourClass) .invoke(includes, target, flavour) as Provider } diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index e9f99602c..cdce9f14b 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -24,7 +24,6 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.konan.target.KonanTarget.* import org.jetbrains.kotlin.konan.target.linker -import python.multiplatform.gradle.CPythonFlavour import python.multiplatform.gradle.CPythonIncludeDirectories import python.multiplatform.gradle.CPythonIncludeLayout import python.multiplatform.gradle.EmbeddedPythonVersion @@ -63,7 +62,18 @@ plugins { val configuredPythonVersion = project.property("pythonVersion").toString() -val pythonFreeThreaded = project.findProperty("pythonFreeThreaded")?.toString()?.toBoolean() ?: false +// Issue #158, maintainer 2026-10-04: "Free-threaded is the default and the only build. The GIL +// build is not kept as an option. Baseline is CPython 3.15t." Desktop (JVM, macosArm64, linuxX64) +// therefore always acquires python-build-standalone's free-threaded build, and the +// `-PpythonFreeThreaded` switch that used to choose it is gone. Android and iOS still embed +// python.org's GIL build until #328, wasm until #175; which build each platform embeds is the +// `embedsFreeThreadedPython` expect/actual in the library, not a build-wide flag. +if (project.findProperty("pythonFreeThreaded") != null) { + throw GradleException( + "-PpythonFreeThreaded was removed (issue #158): desktop always embeds the free-threaded " + + "CPython, and the GIL desktop build is not kept as an option. Remove the property.", + ) +} val pbsRelease = project.findProperty("pythonBuildStandaloneRelease")?.toString() ?: "20261013" val pythonAppleSupportBuild = project.findProperty("pythonAppleSupportBuild")?.toString() ?: "b11" @@ -78,18 +88,16 @@ buildkonfig { defaultConfigs { buildConfigField(FieldSpec.Type.STRING, "pythonVersion", pythonVersion) buildConfigField(FieldSpec.Type.STRING, "libraryVersion", libraryVersion) - // The free-threaded build is not merely a differently-compiled binary: it renames every - // artefact the runtime looks up by name. The shared library is `libpython3.14t.dylib`, not - // `libpython3.14.dylib`, and the stdlib sits in `lib/python3.14t/`. Without this flag - // reaching Kotlin, `manager.loadLibPython` asks for a file that a free-threaded install - // does not contain, and the failure looks like a missing download rather than a flavour - // mismatch. See `Versions.abiFlags`. - buildConfigField(FieldSpec.Type.BOOLEAN, "pythonFreeThreaded", pythonFreeThreaded.toString()) + // No free-threaded field: which build a platform embeds differs by platform (#158), so it + // is the `embedsFreeThreadedPython` expect/actual (commonMain `EmbeddedPythonBuild.kt`), + // which `Versions.abiFlags` reads. One BuildConfig field shared by every target could not + // say "desktop 3.15t, Android 3.15". } } val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".") -val abiSuffix = if (pythonFreeThreaded) "t" else "" +/** ABI flags of the desktop CPython (macOS, Linux, Windows; JVM and Kotlin/Native): `t`, free-threaded (#158). */ +val desktopAbiSuffix = "t" /** * The `ftp/python//` directory holding a release, which drops any pre-release suffix: @@ -114,7 +122,6 @@ println() val licensePath = "src/nativeInterop/cinterop/license" val libPath = "src/nativeInterop/cinterop/lib" -val libPathForDesktop = "$libPath/desktop" val downloadDir = layout.buildDirectory.dir("python-standalone").get().asFile @@ -132,13 +139,14 @@ val downloadDir = layout.buildDirectory.dir("python-standalone").get().asFile val extractedDir = file("$downloadDir/extracted/$configuredPythonVersion") /** - * Suffix that keeps the two desktop flavours apart within [extractedDir]. + * Suffix of the desktop trees within [extractedDir]. * - * The GIL and free-threaded builds of the *same* version are different trees containing - * differently-named libraries (`libpython3.14.dylib` vs `libpython3.14t.dylib`), so they need - * separating for the same reason the version does. + * Desktop acquires only the free-threaded build (#158). The suffix stays in the path because the + * GIL and free-threaded builds of the *same* version are different trees containing + * differently-named libraries (`libpython3.15.dylib` vs `libpython3.15t.dylib`), and a GIL tree + * unpacked by an older checkout must never be mistaken for the free-threaded one. */ -val desktopFlavourSuffix = if (pythonFreeThreaded) "-freethreaded" else "" +val desktopFlavourSuffix = "-freethreaded" /** * Fails the build when the extracted CPython's own headers report a version other than @@ -464,7 +472,7 @@ val desktopTargets = mapOf( ) val downloadTasks = desktopTargets.map { (platform, pbsTarget) -> - val flavour = if (pythonFreeThreaded) "freethreaded-install_only_stripped" else "install_only_stripped" + val flavour = "freethreaded-install_only_stripped" val assetName = "cpython-$configuredPythonVersion+$pbsRelease-$pbsTarget-$flavour.tar.gz" val url = "https://github.com/astral-sh/python-build-standalone/releases/download/$pbsRelease/$assetName" val archive = file("$downloadDir/$assetName") @@ -683,8 +691,8 @@ val downloadAllPythonBuilds by tasks.registering { * target and flavour, each carrying its extraction task. Consumers (pypackpack's compile slot, * TypedPython's C build) read it with * `project(":python-multiplatform").extensions.getByType().includeDir("macos-aarch64")` - * after `evaluationDependsOn(":python-multiplatform")`. Only the desktop flavour selected with - * `-PpythonFreeThreaded` has a task; Android and iOS are always GIL. See + * after `evaluationDependsOn(":python-multiplatform")`. Only the flavour the library embeds has a + * task: free-threaded on desktop (#158), GIL on Android and iOS (#328). See * `docs/platforms/python-version-acquisition.md` §7. */ extensions.add( @@ -693,9 +701,7 @@ extensions.add( layout.projectDirectory.dir(extractedDir.absolutePath), configuredPythonVersion, ) { target, flavour -> - val desktop = target in CPythonIncludeLayout.desktopTargets - val extractedFlavour = if (desktop && pythonFreeThreaded) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL - if (flavour == extractedFlavour) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null + if (flavour == CPythonIncludeLayout.embeddedFlavour(target)) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null }, ) @@ -704,13 +710,29 @@ extensions.add( * extension, as Gradle attributes on the published variants, and as a classpath resource * (`META-INF/python-multiplatform/python.properties`) in the jar and the AAR. See * `docs/platforms/python-version-acquisition.md` "Published version". + * + * Per platform since #158: the desktop jar (and the macosArm64/linuxX64 klibs) embed 3.15t, the AAR, + * the iOS and androidNative klibs the GIL 3.15 until #328, wasm GIL until #175. So the + * free-threaded attribute is decided per published variant, and the jar and the AAR each get + * their own resource. Shared metadata variants carry no CPython and get no free-threaded attribute. + * The `pythonMultiplatform` extension holds one value; it is the desktop one (what composite-build + * consumers that compile against the desktop CPython read), and the per-variant attributes are the + * per-platform truth. (Extension and metadata rules: agent proposal, not confirmed.) */ -val embeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, pythonFreeThreaded) -embeddedPythonVersion.register(project) +val desktopEmbeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = true) +val androidEmbeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = false) +val freeThreadedVariantPrefixes = listOf("desktop", "macosArm64", "linuxX64") +desktopEmbeddedPythonVersion.register(project) { configurationName -> + when { + freeThreadedVariantPrefixes.any { configurationName.startsWith(it) } -> true + configurationName.startsWith("metadata") || "Metadata" in configurationName -> null + else -> false + } +} -val generatePythonProperties by tasks.registering { - val outputDir = layout.buildDirectory.dir("generated/python-properties") - val content = embeddedPythonVersion.renderProperties() +fun registerPythonProperties(name: String, embedded: EmbeddedPythonVersion) = tasks.register(name) { + val outputDir = layout.buildDirectory.dir("generated/python-properties/$name") + val content = embedded.renderProperties() inputs.property("content", content) outputs.dir(outputDir) doLast { @@ -720,6 +742,12 @@ val generatePythonProperties by tasks.registering { } } +/** `freeThreaded=true`: the desktop jar's resource. */ +val generateDesktopPythonProperties = registerPythonProperties("generateDesktopPythonProperties", desktopEmbeddedPythonVersion) + +/** `freeThreaded=false`: the AAR's resource (GIL until #328). */ +val generateAndroidPythonProperties = registerPythonProperties("generateAndroidPythonProperties", androidEmbeddedPythonVersion) + /** * The dev agent (SPEC L-12, #233) is Python, kept as a real file -- `src/commonMain/python/ * python_multiplatform_devagent.py`, which `src/test/python/test_devagent.py` tests with plain CPython -- and @@ -1215,8 +1243,6 @@ kotlin { include("linux-*$desktopFlavourSuffix/python/lib/libpython*.so*") include("windows-*$desktopFlavourSuffix/python/python*.dll") include("windows-*$desktopFlavourSuffix/python/vcruntime*.dll") - // `macos-*` matches `macos-aarch64-freethreaded` too. - if (!pythonFreeThreaded) exclude("*-freethreaded/**") eachFile { if (file.length() == 0L) { exclude(); return@eachFile } val platform = relativePath.segments[0].removeSuffix(desktopFlavourSuffix) @@ -1227,20 +1253,13 @@ kotlin { } } tasks.withType().matching { it.name == "desktopJar" }.configureEach { - if (configuredPythonVersion == "3.13.0" && !pythonFreeThreaded) { - from(libPathForDesktop) { - include("windows-*/*") - include("linux-*/*") - include("macos-*/*") - into("lib") - } - } else { - // Issue #74: the libraries are packed from the staged tree, not straight from the - // extraction, so that symlinks are resolved and the linux library is stripped. - dependsOn(stageDesktopLibraries) - from(stageDesktopLibraries.map { it.destinationDir }) { - include("lib/**") - } + // Issue #74: the libraries are packed from the staged tree, not straight from the + // extraction, so that symlinks are resolved and the linux library is stripped. (A + // branch for checked-in 3.13.0 GIL binaries under `lib/desktop` used to sit here; the + // desktop build is free-threaded only since #158, so it could not be reached.) + dependsOn(stageDesktopLibraries) + from(stageDesktopLibraries.map { it.destinationDir }) { + include("lib/**") } } } @@ -1273,7 +1292,7 @@ kotlin { val targetIncludePath = when(konanTarget.family) { Family.ANDROID -> "$targetExtractDir/include/python$libVersion" Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers" - Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$abiSuffix" + Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$desktopAbiSuffix" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } @@ -1351,7 +1370,7 @@ kotlin { } else if (konanTarget.family == Family.OSX) { getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( "-L$targetExtractDir/lib", - "-lpython$libVersion$abiSuffix", + "-lpython$libVersion$desktopAbiSuffix", "-Wl,-rpath,$targetExtractDir/lib" )) } else if (konanTarget.family == Family.LINUX) { @@ -1363,7 +1382,7 @@ kotlin { linkerOpts.addAll(listOf( "-L$stagedLinuxLibDir", "-L$targetExtractDir/lib", - "-lpython$libVersion$abiSuffix", + "-lpython$libVersion$desktopAbiSuffix", "-Wl,-rpath,$targetExtractDir/lib" )) linkTaskProvider.configure { @@ -1389,7 +1408,7 @@ kotlin { jvmTest.dependsOn(commonTest) val desktopMain by getting { resources.srcDirs("src/desktopMain/resources") - resources.srcDir(generatePythonProperties) + resources.srcDir(generateDesktopPythonProperties) // `reachability-metadata.json` is generated, not checked in -- see // GenerateReachabilityMetadata above. Wiring it as a resource srcDir (through the // task's own output provider, so the dependency is inferred) is what puts it on the @@ -2742,12 +2761,12 @@ android { sourceSets["main"].assets.srcDirs("src/androidMain/assets", "$androidBuildDir/assets") // KMP androidMain resources do not reach the AAR's classes.jar; AGP's main resources do. - sourceSets["main"].resources.srcDir(generatePythonProperties) + sourceSets["main"].resources.srcDir(generateAndroidPythonProperties) // AGP's legacy source-set API resolves that provider to files and drops the task dependency // (issue #72: `processReleaseJavaRes` read the output with no declared dependency), so it is // declared on every AGP java-resources task explicitly. tasks.matching { Regex("""process\w*JavaRes""").matches(it.name) } - .configureEach { dependsOn(generatePythonProperties) } + .configureEach { dependsOn(generateAndroidPythonProperties) } sourceSets["androidTest"].assets.srcDirs("$androidBuildDir/assets") sourceSets["debug"].jniLibs.srcDirs("src/androidMain/jniLibs", "$androidBuildDir/jniLibs", "$androidBuildDir/debug/jniLibs") diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt new file mode 100644 index 000000000..b9a3cc45c --- /dev/null +++ b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt @@ -0,0 +1,34 @@ +package python.multiplatform + +import python.multiplatform.env.PythonBootstrap +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Issue #158 moved desktop to the free-threaded CPython 3.15t; Android (ART) still embeds + * python.org's GIL CPython 3.15 until #328. This pins that the desktop change did not leak here. + * + * Not red before the change: Android was GIL before and is GIL after. It is the guard against the + * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which + * [PythonBootstrap.stdlibRelativePath] becomes `lib/python3.15t`, a directory the staged GIL + * prefix does not contain, and the app dies in `onCreate`. Nothing here loads the interpreter. It + * lives in `androidInstrumentedTest` because `androidUnitTest` does not compile on develop (four + * `commonTest` expects have no host-JVM actual), not because it needs a device. + */ +class AndroidGilBuildTest { + + @Test + fun androidStillReportsTheGilAbi() { + val v = Versions.currentVersion + assertEquals("", v.abiFlags, "Android embeds the GIL CPython until #328") + assertEquals(v.compactVersionString, v.taggedVersionString) + } + + @Test + fun theStdlibIsLookedForUnderTheGilDirectory() { + val compact = Versions.currentVersion.compactVersionString + assertEquals("lib/python$compact", PythonBootstrap.stdlibRelativePath()) + assertEquals(".python-multiplatform-stdlib.$compact", PythonBootstrap.stampFile(File("/prefix")).name) + } +} diff --git a/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt new file mode 100644 index 000000000..dd8d5e0fa --- /dev/null +++ b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** Android (ART) embeds python.org's GIL CPython until #328. See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = false diff --git a/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt new file mode 100644 index 000000000..79200f5e8 --- /dev/null +++ b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** androidNative links python.org's GIL CPython until #328. See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = false diff --git a/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt new file mode 100644 index 000000000..ca41ade40 --- /dev/null +++ b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt @@ -0,0 +1,22 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Issue #158 moved desktop to the free-threaded CPython 3.15t; androidNative still embeds the GIL build until + * #328. This pins that the desktop change did not leak here. + * + * Not red before the change: androidNative was GIL before and is GIL after. It is the guard against the + * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which + * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that androidNative does not ship. + */ +class AndroidNativeGilBuildTest { + + @Test + fun androidNativeStillReportsTheGilAbi() { + val v = Versions.currentVersion + assertEquals("", v.abiFlags, "androidNative embeds the GIL CPython until #328") + assertEquals(v.compactVersionString, v.taggedVersionString) + } +} diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt new file mode 100644 index 000000000..9d2c69b44 --- /dev/null +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt @@ -0,0 +1,30 @@ +package python.multiplatform + + +/** + * Whether the CPython this platform embeds is a free-threaded build (PEP 703, `Py_GIL_DISABLED`). + * + * A property of the platform, not of the whole build. The maintainer's decision on issue #158 + * (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an + * option. Baseline is CPython 3.15t." The desktop half landed first (leader, 2026-11-13: "flip the + * desktop default to 3.15t now, before the #377 alpha03 release"); Android and iOS still embed + * the GIL CPython 3.15 they download from python.org until #328, and wasm until #175. So: + * + * | Source set | Targets | Embeds | + * |---------------------|---------------------------------|---------------| + * | `desktopMain` | desktop JVM | 3.15t | + * | `desktopNativeMain` | `macosArm64`, `linuxX64` | 3.15t | + * | `androidMain` | Android (ART) | 3.15 (GIL) | + * | `artMain` | `androidNativeArm64`, `X64` | 3.15 (GIL) | + * | `iosMain` | iOS | 3.15 (GIL) | + * | `wasmJsMain` | `wasmJs` | GIL | + * + * It used to be one `BuildConfig.pythonFreeThreaded` shared by every target, which could not + * express this: flipping it for desktop made Android look for `lib/python3.15t`, a directory its + * GIL prefix does not have. A plain (non-`inline`) `expect val`, so the intermediate-source-set + * crash in `commonMain/README.md` does not apply. + * + * Everything CPython names after itself follows from this through [Versions.abiFlags]: the shared + * library (`libpython3.15t`), the stdlib directory (`lib/python3.15t/`), and the PYTHONHOME check. + */ +internal expect val embedsFreeThreadedPython: Boolean diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt index 49cbc440c..b8cc66d2e 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt @@ -50,12 +50,13 @@ class Versions private constructor(val versionString: String) { /** * CPython's ABI flags suffix: `t` on a free-threaded build (PEP 703), empty otherwise. * - * This is a property of the *build*, not of the version -- 3.14 ships in both flavours out of - * the same source tree -- so it comes from [BuildConfig.pythonFreeThreaded] rather than from - * [versionString]. + * This is a property of the *build*, not of the version -- 3.15 ships in both flavours out of + * the same source tree -- and of the *platform*: desktop (JVM and Kotlin/Native) embeds 3.15t, + * while Android and iOS still embed the GIL build until #328 (issue #158). So it comes from the + * per-platform [embedsFreeThreadedPython] rather than from [versionString]. */ val abiFlags: String - get() = if (BuildConfig.pythonFreeThreaded) "t" else "" + get() = if (embedsFreeThreadedPython) "t" else "" /** * `major.minor` plus [abiFlags]: `3.14` on a default build, `3.14t` free-threaded. diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index 5060b9469..d31fac6a9 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -1,6 +1,6 @@ package python.multiplatform.ffi -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.env.PythonHomeCheck import python.multiplatform.env.PythonDevAgent import python.multiplatform.env.PythonPayload @@ -809,6 +809,10 @@ def __pmp_run_file__(path, argv): * to take none automatically. A checkpoint is additionally skipped when nothing has been * released since the previous one, so a workload that never drops a wrapper never pays. * + * Which build that is depends on the platform ([python.multiplatform.embedsFreeThreadedPython]): + * desktop embeds 3.15t since issue #158, so this defaults to on there; Android and iOS embed the + * GIL build until #328, so it defaults to off there. + * * Defaults to on for free-threaded builds, where deferred release is the defect described on * [drainPendingReleases], a build with the global lock frees a *plain* reference the * cleaner's `Py_DecRef` immediately, with no queue to drain, so that half of the checkpoint's @@ -827,7 +831,7 @@ def __pmp_run_file__(path, argv): * call [PyGC_Collect] directly for an immediate, unconditional collection, [drainPendingReleases] * and `PyGC_Collect` both work regardless of this setting. */ - var autoDrainInterval: Int = if (BuildConfig.pythonFreeThreaded) DEFAULT_AUTO_DRAIN_INTERVAL else 0 + var autoDrainInterval: Int = if (embedsFreeThreadedPython) DEFAULT_AUTO_DRAIN_INTERVAL else 0 /** Diagnostic counters for the checkpoint machinery. Test-visible. */ internal object CheckpointCounter { diff --git a/python-multiplatform/src/commonTest/README.md b/python-multiplatform/src/commonTest/README.md index 60bcf485b..b7e031387 100644 --- a/python-multiplatform/src/commonTest/README.md +++ b/python-multiplatform/src/commonTest/README.md @@ -13,8 +13,8 @@ platform-only file actually needs to be there, and one promotion that was tried | Execution path | Gradle task | Source-set chain | |---|---|---| -| desktop | `:python-multiplatform:desktopTest` | `desktopTest -> jvmTest -> commonTest` | -| desktop, free-threaded | same task, `-PpythonFreeThreaded=true` | same chain, linked against `libpython3.14t` instead of `libpython3.14` | +| desktop | `:python-multiplatform:desktopTest` | `desktopTest -> jvmTest -> commonTest`, against the free-threaded `libpython3.15t` (the only desktop build since #158) | +| desktop native | `:python-multiplatform:macosArm64Test`, `linuxX64Test` | `{macosArm64Test, linuxX64Test} -> desktopNativeTest -> nativeTest -> commonTest`, linked against `libpython3.15t` | | iOS simulator | `:python-multiplatform:iosSimulatorArm64Test` | `iosSimulatorArm64Test -> nativeTest -> commonTest` | | androidNative | `:python-multiplatform:androidNative{Arm64,X64}Test` (custom `adb`-driving task; `compileTestKotlinAndroidNativeArm64` for a device-free check) | `androidNative*Test -> artTest -> nativeTest -> commonTest` | | Android (ART) | `connectedAndroidTest` (instrumented, needs a device/emulator) | `androidInstrumentedTest -> jvmTest -> commonTest` | diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt index c64073c1b..0cdc3aa8f 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt @@ -13,15 +13,19 @@ import kotlin.test.assertTrue * therefore does not degrade gracefully: `manager.loadLibPython` throws `UnsatisfiedLinkError` for * a file that was downloaded and is sitting right there under a different name. * - * These assertions are written against [BuildConfig.pythonFreeThreaded] rather than hard-coding one - * flavour, so the same suite is meaningful under `-PpythonFreeThreaded=true`. + * These assertions are written against the per-platform [embedsFreeThreadedPython] rather than + * hard-coding one flavour, because the flavour differs by platform (issue #158): desktop embeds + * 3.15t, Android and iOS the GIL 3.15 until #328. The platform test source sets pin the literal + * value for their platform (`DesktopFreeThreadedBuildTest`, `DesktopNativeFreeThreadedBuildTest`, + * `AndroidGilBuildTest`, `AndroidNativeGilBuildTest`, `IosGilBuildTest`), which is what a wrong + * actual would fail. */ class VersionsTest { @Test fun abiFlagsFollowsTheConfiguredFlavour() { assertEquals( - if (BuildConfig.pythonFreeThreaded) "t" else "", + if (embedsFreeThreadedPython) "t" else "", Versions.currentVersion.abiFlags, "abiFlags must be CPython's flags suffix: 't' free-threaded (PEP 703), empty otherwise" ) @@ -31,7 +35,7 @@ class VersionsTest { fun taggedVersionStringNamesTheArtefactsOnDisk() { val v = Versions.currentVersion assertEquals(v.compactVersionString + v.abiFlags, v.taggedVersionString) - if (BuildConfig.pythonFreeThreaded) { + if (embedsFreeThreadedPython) { assertTrue( v.taggedVersionString.endsWith("t"), "a free-threaded build must ask for libpython${v.compactVersionString}t, " + diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt index 3ef23f66a..59d2b236d 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.ReleaseCounter @@ -210,7 +210,7 @@ class EvalCheckpointTest { // With the global lock, `refNow` is the reading that belongs to this test and // to nothing else, so it is not enough for the counter alone to be satisfied. cleanerReleases >= OUTER && - (BuildConfig.pythonFreeThreaded || refNow <= refBefore) + (embedsFreeThreadedPython || refNow <= refBefore) } }, finish = { try { @@ -234,7 +234,7 @@ class EvalCheckpointTest { val refAfterCleaner = refCount(target) - if (BuildConfig.pythonFreeThreaded) { + if (embedsFreeThreadedPython) { assertEquals( refAfterBuild, refAfterCleaner, @@ -319,7 +319,7 @@ class EvalCheckpointTest { val gcCollectNs = timeNs(100) { withGIL { PyGC_Collect() } } println( - "\n--- eval-loop checkpoint cost (freeThreaded=${BuildConfig.pythonFreeThreaded}) ---\n" + + "\n--- eval-loop checkpoint cost (freeThreaded=${embedsFreeThreadedPython}) ---\n" + "withGIL { } ${emptyScopeNs.format()} ns/op\n" + "Python3.drainPendingReleases() ${drainNs.format()} ns/op\n" + "withGIL { Py_MakePendingCalls } ${makePendingNs.format()} ns/op (does NOT merge)\n" + diff --git a/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt b/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt new file mode 100644 index 000000000..ad134981a --- /dev/null +++ b/python-multiplatform/src/desktopMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktop.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** Desktop JVM embeds python-build-standalone's free-threaded CPython (issue #158). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt new file mode 100644 index 000000000..7c5453452 --- /dev/null +++ b/python-multiplatform/src/desktopNativeMain/kotlin/python/multiplatform/EmbeddedPythonBuild.desktopNative.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** macosArm64 and linuxX64 link python-build-standalone's free-threaded CPython (issue #158). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt new file mode 100644 index 000000000..2e9c36f80 --- /dev/null +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt @@ -0,0 +1,23 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-1 (issue #158): desktop Kotlin/Native (`macosArm64`, `linuxX64`) links the free-threaded + * CPython 3.15t, the same build as the desktop JVM. + * + * Red before the change: `abiFlags` came from the one `BuildConfig.pythonFreeThreaded` (default + * false), so it was `""` here, and the cinterop compiled against `include/python3.15` and linked + * `-lpython3.15`. This source set shares `nativeMain` with iOS and androidNative, which must stay + * GIL (`IosGilBuildTest`, `AndroidNativeGilBuildTest`), so a single shared flag cannot pass both. + */ +class DesktopNativeFreeThreadedBuildTest { + + @Test + fun desktopNativeReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "desktop Kotlin/Native links CPython ${v.compactVersionString}t (#158)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt new file mode 100644 index 000000000..990f00e23 --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt @@ -0,0 +1,55 @@ +package python.multiplatform + +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * SPEC T-1 (issue #158): desktop embeds the free-threaded CPython 3.15t, and only that build. + * + * The maintainer's decision on #158 (2026-10-04): "Free-threaded is the default and the only build. + * The GIL build is not kept as an option. Baseline is CPython 3.15t." Desktop moved first (leader, + * 2026-11-13); Android and iOS follow under #328. + * + * Red before the change: the desktop build was GIL unless `-PpythonFreeThreaded=true` was passed, + * so `abiFlags` was `""`, the tagged version was `3.15`, the interpreter reported + * `Py_GIL_DISABLED = 0` and `autoDrainInterval` defaulted to 0. Each assertion below fails on that + * build with its own message. + * + * What this cannot find: whether Android or iOS were left alone. Their literal values are pinned in + * their own test source sets (`AndroidGilBuildTest`, `AndroidNativeGilBuildTest`, `IosGilBuildTest`). + */ +class DesktopFreeThreadedBuildTest { + + @Test + fun desktopReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "desktop embeds CPython ${v.compactVersionString}t (#158)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLoadedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + // The claim above is only worth something if the libpython that `manager.loadLibPython` + // actually loaded agrees with it. `Py_GIL_DISABLED` is a build-time fact of that library; + // `sys._is_gil_enabled()` is not asserted, because an extension module without + // `Py_mod_gil` re-enables the GIL at run time on a free-threaded build. + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the desktop libpython is a GIL build; desktop must embed the free-threaded one (#158)", + ) + } + + @Test + fun theAutomaticCheckpointIsOnByDefault() { + // `Python3.autoDrainInterval` is a mutable default that other tests save and restore; + // a positive value is what the free-threaded default means (Python3.kt). + assertTrue( + Python3.autoDrainInterval > 0, + "autoDrainInterval=${Python3.autoDrainInterval}: the free-threaded default turns the checkpoint on", + ) + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt index 9ce815401..4e7647f44 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt @@ -5,7 +5,16 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -/** Issue #61: the published jar carries the CPython version it embeds, agreeing with BuildConfig. */ +/** + * Issue #61: the published jar carries the CPython version it embeds, agreeing with BuildConfig. + * + * Issue #158: the desktop jar embeds the free-threaded 3.15t, so its resource says + * `freeThreaded=true`. The AAR carries its own resource (`freeThreaded=false`, GIL until #328); + * the two are generated separately in `python-multiplatform/build.gradle.kts`. + * + * Red before #158: one resource served the jar and the AAR, rendered from + * `-PpythonFreeThreaded` (default false), so the desktop jar said `freeThreaded=false`. + */ class PublishedPythonVersionResourceTest { // Same path as resourcePath; that class lives in the build plugin and is // not on the test classpath. @@ -18,6 +27,6 @@ class PublishedPythonVersionResourceTest { val props = stream.use { it.readBytes().decodeToString() }.lines() .filter { '=' in it }.associate { it.substringBefore('=').trim() to it.substringAfter('=').trim() } assertEquals(BuildConfig.pythonVersion, props["pythonVersion"]) - assertEquals(BuildConfig.pythonFreeThreaded, props["freeThreaded"].toBoolean()) + assertEquals("true", props["freeThreaded"], "the desktop jar embeds the free-threaded CPython (#158)") } } diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt index d0dd7e3c0..16ff5453f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/PyTypeCacheConcurrencyTest.kt @@ -18,8 +18,8 @@ import kotlin.test.assertTrue * Before the copy-on-write cache the memo was a plain `mutableMapOf`. Two threads putting at once * could lose an entry (a later lookup then builds a second [PyType] for the same pointer, so * identity differs across threads) or corrupt the map outright (an exception). That needs real - * parallel writers, so it can fail only on a free-threaded CPython (`-PpythonFreeThreaded=true`); - * under the default GIL, getInstance's own `withPython` calls serialise most of the interleaving + * parallel writers, so it can fail only on a free-threaded CPython (desktop's build since issue + * #158); under a GIL build, getInstance's own `withPython` calls serialise most of the interleaving * and the test is a regression guard rather than a reproducer. * * Asserted: no thread throws; for each type every thread was handed the same instance; a final diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 99493fe05..c3f33358f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -15,7 +15,7 @@ import kotlin.test.BeforeTest import python.native.ffi.bindings import python.native.ffi.Panama import python.native.ffi.toNativePointer -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython class Node(var ref: PyObject? = null) { var rawPtr: Long = 0L @@ -146,7 +146,7 @@ class CycleCollectionTest { * with the global lock, and why the free-threaded branch is not just an offset fix. */ private fun typeRefCount(addr: Long): Long = - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { getLongMethod.invoke(theUnsafe, addr) as Long } else { val local = (getIntMethod.invoke(theUnsafe, addr + 12) as Int).toLong() and 0xFFFF_FFFFL @@ -383,7 +383,7 @@ class CycleCollectionTest { // `Py_REFCNT`, for whichever object header this build actually has. See the KDoc above: // offset 0 is `ob_refcnt` only on a build with the global lock. fun typeRefCount(): Long = - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { getLong.invoke(theUnsafe, proxyTypeAddr) as Long } else { val local = (getInt.invoke(theUnsafe, proxyTypeAddr + 12) as Int).toLong() and 0xFFFF_FFFFL @@ -395,7 +395,7 @@ class CycleCollectionTest { // carries, so that `before` and `alive` are read in the same state and the difference // between them is the instantiation loop and nothing else. On a build with the global // lock the count is always accurate and this would only cost a heap walk. - if (BuildConfig.pythonFreeThreaded) python.native.ffi.PyGC_Collect() + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount val typeRefBefore = typeRefCount() @@ -421,7 +421,7 @@ class CycleCollectionTest { ) // The instantiations above were deferred on a free-threaded build; this is what turns // them into countable references. Without it `alive` reads exactly `before`. - if (BuildConfig.pythonFreeThreaded) python.native.ffi.PyGC_Collect() + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val typeRefAlive = typeRefCount() assertEquals( diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt index d19b2dae0..38b439ec4 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.PythonTestFixture @@ -113,7 +113,7 @@ class FreeThreadedGCGateTest { buildAndDropCycles(builtins, cyclesCreated / 2) val residue = countTrackedObjects(gc) - initial println( - "GATE[$label] freeThreaded=${BuildConfig.pythonFreeThreaded} residue=$residue/$totalObjects " + + "GATE[$label] freeThreaded=${embedsFreeThreadedPython} residue=$residue/$totalObjects " + "collectionsThatActuallyRan=${gen0Collections(gc) - collectionsBefore}" ) forceCollect(gc) @@ -162,7 +162,7 @@ class FreeThreadedGCGateTest { "short-circuits gc_should_collect on old[0].threshold == 0" ) - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { // gc.c:1866 schedules on generations[0].count alone and gc.c:1258 selects // generation 0 on count alone, so generation 1's threshold cannot matter. assertTrue( @@ -220,7 +220,7 @@ class FreeThreadedGCGateTest { Python3.exec("del __pm_ballast") val dropped = arm(gc, builtins, "ballast-dropped") - if (BuildConfig.pythonFreeThreaded) { + if (embedsFreeThreadedPython) { // The whole finding: JVM memory growth, and nothing else, collected Python // cycles. Deliberately not a tight bound -- how much a single collection // takes depends on where in the workload the footprint crossed the bar. diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt index d08904416..fb3863b9a 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/GCSchedulingMeasurementTest.kt @@ -1,6 +1,6 @@ package python.multiplatform.ref -import python.multiplatform.BuildConfig +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.Python3 import python.multiplatform.ffi.PythonTestFixture import kotlin.test.Test @@ -11,9 +11,11 @@ import kotlin.test.assertTrue * build specifically: `_PY_GC_SCHEDULED_BIT` is read only by the eval-loop checkpoint, on both * builds, so an embedder that never runs Python bytecode never runs the cyclic collector either * -- with or without free threading. `autoDrainInterval` is saved and restored around each test; - * the default (0 on this build) is never changed by running this file. + * the default (0 on a GIL build, 32 on the free-threaded one) is never changed by running this file. * - * Also runs, unmodified, on `-PpythonFreeThreaded=true`, where the automatic checkpoint reliably + * Desktop embeds the free-threaded build since issue #158, so the GIL branches below no longer run + * on desktop; they are kept for the record of what the GIL build measured. On the free-threaded + * build the automatic checkpoint reliably * fires but does not reliably reclaim: under this heavy single-thread, C-API-only workload the * residue swings between the GIL build's ~1,970 and the full ~20,000, run to run, so only the GIL * build gets a bound here. That swing is not a mystery any more -- it tracks the *JVM's* memory @@ -165,7 +167,7 @@ class GCSchedulingMeasurementTest { * The residue bound is pinned on the GIL build only, where it is stable at ~1,920-1,970 out of * ~20,000 in every configuration measured. * - * Free-threaded (`-PpythonFreeThreaded=true`, where `autoDrainInterval = 32` is already the + * Free-threaded (desktop's only build since issue #158, where `autoDrainInterval = 32` is already the * shipped default rather than something this test turns on) it is not stable, and no bound is * asserted. `CheckpointCounter.reached` climbs by an identical, deterministic amount on every * run, but the residue does not: measured 20,000 -- i.e. nothing reclaimed at all -- in 10 of @@ -222,7 +224,7 @@ class GCSchedulingMeasurementTest { val delta = after - initial println( - "--- MEASUREMENT 2: Interval 32 (freeThreaded=${BuildConfig.pythonFreeThreaded}) --- " + + "--- MEASUREMENT 2: Interval 32 (freeThreaded=${embedsFreeThreadedPython}) --- " + "initial=$initial after=$after delta=$delta checkpointsReached=$reachedDelta" ) @@ -232,7 +234,7 @@ class GCSchedulingMeasurementTest { "$cyclesCreated outermost withGIL scopes, but none were reached" ) - if (!BuildConfig.pythonFreeThreaded) { + if (!embedsFreeThreadedPython) { // Each outermost withGIL scope inside buildAndDropCycles counts down; at interval // 32 that is thousands of checkpoints over 10,000 rounds, each one able to run the // scheduled collection. The residue should be bounded by roughly one interval's diff --git a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt index 0a1cab1a4..cf05640d6 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/SidecarLibraryLookupTest.kt @@ -28,8 +28,8 @@ class SidecarLibraryLookupTest { "PYTHONHOME is unset for desktopTest; the sidecar lookup has nothing to resolve against" ) - // `taggedVersionString`, not `compactVersionString`: under `-PpythonFreeThreaded=true` - // PYTHONHOME contains `libpython3.14t.dylib` and no `libpython3.14.dylib`, so asking for + // `taggedVersionString`, not `compactVersionString`: desktop embeds the free-threaded + // build (issue #158), so PYTHONHOME contains `libpython3.15t.dylib` and no `libpython3.15.dylib`, so asking for // the un-suffixed name resolves nothing. This mirrors what `manager.loadLibPython` builds. val tagged = Versions.currentVersion.taggedVersionString val libraryName = if (currentPlatform.os == OSType.Windows) { diff --git a/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt new file mode 100644 index 000000000..475ae94f8 --- /dev/null +++ b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** iOS embeds the GIL Python.xcframework until #328. See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = false diff --git a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt new file mode 100644 index 000000000..8c581259f --- /dev/null +++ b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt @@ -0,0 +1,22 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Issue #158 moved desktop to the free-threaded CPython 3.15t; iOS still embeds the GIL build until + * #328. This pins that the desktop change did not leak here. + * + * Not red before the change: iOS was GIL before and is GIL after. It is the guard against the + * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which + * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that iOS does not ship. + */ +class IosGilBuildTest { + + @Test + fun iosStillReportsTheGilAbi() { + val v = Versions.currentVersion + assertEquals("", v.abiFlags, "iOS embeds the GIL CPython until #328") + assertEquals(v.compactVersionString, v.taggedVersionString) + } +} diff --git a/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt b/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt new file mode 100644 index 000000000..798ce6f03 --- /dev/null +++ b/python-multiplatform/src/wasmJsMain/kotlin/python/multiplatform/EmbeddedPythonBuild.wasmJs.kt @@ -0,0 +1,4 @@ +package python.multiplatform + +/** wasmJs runs a GIL CPython built for wasm32-emscripten until #175. See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = false diff --git a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt new file mode 100644 index 000000000..06a9dfc56 --- /dev/null +++ b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt @@ -0,0 +1,22 @@ +package python.multiplatform + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Issue #158 moved desktop to the free-threaded CPython 3.15t; wasmJs still embeds the GIL build until + * #175. This pins that the desktop change did not leak here. + * + * Not red before the change: wasmJs was GIL before and is GIL after. It is the guard against the + * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which + * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that wasmJs does not ship. + */ +class WasmGilBuildTest { + + @Test + fun wasmStillReportsTheGilAbi() { + val v = Versions.currentVersion + assertEquals("", v.abiFlags, "wasmJs embeds the GIL CPython until #175") + assertEquals(v.compactVersionString, v.taggedVersionString) + } +} diff --git a/sample/build.gradle.kts b/sample/build.gradle.kts index 326e10cb7..0b8d39e0d 100644 --- a/sample/build.gradle.kts +++ b/sample/build.gradle.kts @@ -350,8 +350,8 @@ val hostPlatform = when { // the same way rather than hardcoding a flat path -- otherwise the sample points PYTHONHOME at a // directory that only exists for the default configuration. val samplePythonVersion = project.findProperty("pythonVersion")?.toString() ?: rootProject.version.toString() -val sampleFlavourSuffix = - if (project.findProperty("pythonFreeThreaded")?.toString()?.toBoolean() == true) "-freethreaded" else "" +// Desktop acquires only the free-threaded build (#158), unpacked under `-freethreaded`. +val sampleFlavourSuffix = "-freethreaded" val pythonHomeForHost = project(":python-multiplatform").layout.buildDirectory .dir("python-standalone/extracted/$samplePythonVersion/$hostPlatform$sampleFlavourSuffix/python") From b96aa739f25335e5c323a2f1db44ac5f1cfe155a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:13:30 +0900 Subject: [PATCH 44/61] Android, androidNative and iOS embed free-threaded CPython 3.15t (#328 step 2) Owner decision #158 (2026-10-04; 3.15t is the only build), leader 2026-11-13 (no mobile opt-in), owner 2026-11-13 (arm64-v8a and x86_64 both on 3.15t). The library downloads the project's own 3.15t archives from release cpython-3.15.0t-mobile (pinned), checks Py_GIL_DISABLED in every pyconfig header, packages libpython3.15t.so and lib/python3.15t per ABI, links androidNative against python3.15t and cinterops iOS against the 3.15t xcframework. embedsFreeThreadedPython is true on every platform but wasm. The refcount test probes read the free-threaded object header. GIL mobile pins, the BeeWare/python.org iOS switch and pythonAppleSupportBuild are removed. --- AGENTS.md | 6 +- README.md | 2 +- docs/SPEC.md | 72 ++-- docs/design/threading-and-abi.md | 10 +- docs/locale/README_ko.md | 2 +- docs/platforms/free-threaded-mobile.md | 49 ++- docs/platforms/python-version-acquisition.md | 82 +++-- docs/roadmap/ROADMAP.md | 21 +- gradle.properties | 1 - python-checksums.properties | 7 +- .../fixtures/klib-artifact/build.gradle.kts | 5 +- .../gradle/AcquiredHeaderVersion.kt | 6 +- .../gradle/CPythonIncludeDirectories.kt | 48 +-- .../gradle/EmbeddedPythonVersion.kt | 2 +- .../multiplatform/gradle/IosConsumerWiring.kt | 2 +- .../gradle/AcquiredHeaderVersionTest.kt | 2 +- .../gradle/CPythonIncludeDirectoriesTest.kt | 67 ++-- .../fixtures/app/build.gradle.kts | 7 +- python-multiplatform/build.gradle.kts | 322 +++++++++--------- .../AndroidFreeThreadedBuildTest.kt | 71 ++++ .../multiplatform/AndroidGilBuildTest.kt | 34 -- .../multiplatform/ref/CycleCollectionTest.kt | 15 +- .../EmbeddedPythonBuild.android.kt | 4 +- .../native/ffi/ShapeDowncalls.android.kt | 2 +- .../kotlin/python/native/ffi/bindings.kt | 8 +- .../src/artMain/cinterop/jni_onload.def | 18 +- .../multiplatform/EmbeddedPythonBuild.art.kt | 4 +- .../AndroidNativeFreeThreadedBuildTest.kt | 32 ++ .../AndroidNativeGilBuildTest.kt | 22 -- .../multiplatform/EmbeddedPythonBuild.kt | 19 +- .../kotlin/python/multiplatform/Versions.kt | 6 +- .../python/multiplatform/ffi/Python3.kt | 4 +- .../python/multiplatform/VersionsTest.kt | 11 +- .../DesktopNativeFreeThreadedBuildTest.kt | 4 +- .../DesktopFreeThreadedBuildTest.kt | 5 +- .../env/PublishedPythonVersionResourceTest.kt | 5 +- .../multiplatform/EmbeddedPythonBuild.ios.kt | 4 +- .../multiplatform/IosFreeThreadedBuildTest.kt | 33 ++ .../python/multiplatform/IosGilBuildTest.kt | 22 -- .../python/native/ffi/ShapeDowncalls.kt | 2 +- .../multiplatform/ref/CycleCollectionTest.kt | 55 ++- .../python/multiplatform/WasmGilBuildTest.kt | 4 +- 42 files changed, 642 insertions(+), 455 deletions(-) create mode 100644 python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt delete mode 100644 python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt create mode 100644 python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt delete mode 100644 python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt create mode 100644 python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt delete mode 100644 python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt diff --git a/AGENTS.md b/AGENTS.md index 3b0d921b3..d06dfc06d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -271,8 +271,10 @@ around one. (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t." Desktop (JVM, `macosArm64`, `linuxX64`) embeds 3.15t with no GIL option since the leader's instruction of 2026-11-13 ("flip the desktop default to 3.15t now"); - Android and iOS still embed the GIL 3.15 until #328, wasm until #175. Which build a platform embeds - is the per-platform `embedsFreeThreadedPython`, never one build-wide flag (SPEC T-1, ROADMAP §9). + Android (arm64-v8a and x86_64), androidNative and iOS embed the project's own 3.15t builds with no + opt-in flag (#328; leader decision 2026-11-13, owner approval of the x86_64 build the same day); + wasm embeds a GIL build until #175. Which build a platform embeds is the per-platform + `embedsFreeThreadedPython`, never one build-wide flag (SPEC T-1, ROADMAP §9). 9. **Do not add PanamaPort as a dependency** (licence and dependence on ART internals). ## 13. Upcalls must work in a GraalVM native image diff --git a/README.md b/README.md index 52f756c89..79d37159a 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Honest summary, see the [roadmap](docs/roadmap/ROADMAP.md) for what is left to d | Binding prebuilt jars, Compose from Python | ✅ desktop · ⏳ Android / iOS / wasm | | Binding Kotlin/Native klibs | 🟡 scanner only | | `.pyi` stub generation | 🟡 partial | -| Free-threaded CPython 3.15t (the only desktop build since #158; Android / iOS still GIL until #328) | 🟡 desktop · ⏳ Android / iOS / wasm | +| Free-threaded CPython 3.15t (the only build on desktop since #158 and on Android / iOS since #328; wasm GIL until #175) | 🟡 desktop · 🟡 Android / iOS (compiled, not yet run on emulator or simulator) · ⏳ wasm | ## 🧱 The ecosystem diff --git a/docs/SPEC.md b/docs/SPEC.md index 0e6800982..088c18697 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -649,18 +649,25 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **T-1** Which CPython build each platform embeds. Maintainer decision on #158 (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t." Leader instruction (2026-11-13): "flip the desktop default to 3.15t now, before the #377 alpha03 - release"; mobile follows under #328, wasm under #175. So today: + release"; mobile follows under #328 (leader decision 2026-11-13 on #328: no opt-in flag on mobile, + 3.15t is the only build; the owner approved the x86_64 Android 3.15t build on 2026-11-13, so + arm64-v8a and x86_64 both stay supported), wasm under #175. So today: desktop JVM, `macosArm64` and `linuxX64` embed the free-threaded CPython 3.15t, with no GIL option - (`-PpythonFreeThreaded` is removed and the build fails if it is passed); Android (ART and - androidNative) and iOS embed python.org's GIL CPython 3.15 and wasm a GIL CPython until those - issues land. The fact is per platform (`embedsFreeThreadedPython`, an `expect`/`actual`), and - `Versions.abiFlags` (`t` or empty) and every name derived from it (`libpython3.15t`, `lib/python3.15t/`) - follow it. On the GIL platforms Kotlin threads attach a thread state and serialise on the GIL. - `Status: partial` (desktop only; Android and iOS remain GIL until #328), + (`-PpythonFreeThreaded` is removed and the build fails if it is passed); Android (ART on arm64-v8a + and x86_64, and androidNative) and iOS embed the project's own free-threaded CPython 3.15t (T-2), with + no GIL option either; wasm embeds a GIL CPython until #175. The fact is per platform + (`embedsFreeThreadedPython`, an `expect`/`actual`, kept while wasm is the exception), and + `Versions.abiFlags` (`t` or empty) and every name derived from it (`libpython3.15t`, `lib/python3.15t/`, + `libmultiplatform_python3.15t.so`) follow it. On the GIL platform Kotlin threads attach a thread state + and serialise on the GIL. + `Status: partial` (desktop observed; Android, androidNative and iOS are compiled against 3.15t but + have not been run on an emulator, device or simulator in this revision), `PM/desktopTest/.../DesktopFreeThreadedBuildTest.kt` (ABI flags, `Py_GIL_DISABLED` of the loaded interpreter, checkpoint default), `PM/desktopNativeTest/.../DesktopNativeFreeThreadedBuildTest.kt`, - `PM/androidInstrumentedTest/.../AndroidGilBuildTest.kt`, `PM/artTest/.../AndroidNativeGilBuildTest.kt`, - `PM/iosSimulatorArm64Test/.../IosGilBuildTest.kt`, `PM/wasmJsTest/.../WasmGilBuildTest.kt`, + `PM/androidInstrumentedTest/.../AndroidFreeThreadedBuildTest.kt` (ABI flags, stdlib path, the AAR's + `python.properties`, `Py_GIL_DISABLED` of the loaded interpreter, checkpoint default), + `PM/artTest/.../AndroidNativeFreeThreadedBuildTest.kt`, `PM/iosSimulatorArm64Test/.../IosFreeThreadedBuildTest.kt`, + `PM/wasmJsTest/.../WasmGilBuildTest.kt`, `PM/commonTest/.../VersionsTest.kt`; the release after initialisation is tested by `GilReleaseTest` / `GilReleaseNativeTest` (see C-4). - **T-1a** On desktop, Python may be entered from a JVM virtual thread (#161 option B, user decision): @@ -679,8 +686,18 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. T-1), `GP/PythonHomeStagingTest.kt`, `PM/desktopTest/.../FreeThreadedGCGateTest.kt`, `PM/desktopTest/.../DesktopFreeThreadedBuildTest.kt`. When it was an opt-in, 3.14t ran the whole desktop suite (236 tests, 0 failures; ROADMAP §9); the 3.15t default has not been observed running the - suite in this revision. Android and iOS free-threaded builds are #163/#328. `Py_LIMITED_API` is not - defined, so `abi3t` is not a blocker. + suite in this revision. `Py_LIMITED_API` is not defined, so `abi3t` is not a blocker. + The library's Android and iOS builds (#328 step 2) acquire the project's own free-threaded archives + from the GitHub release `cpython-t-mobile` (#163, `build-ft-mobile.yml`), each verified against + its SHA-256 in `python-checksums.properties` (`android-aarch64--freethreaded`, + `android-x86_64--freethreaded`, `ios--freethreaded`; the release's `SHA256SUMS` agrees) + and against its own headers (`PY_VERSION`, and `Py_GIL_DISABLED 1` in every `pyconfig` header). + Android unpacks under `android--freethreaded/`, links androidNative against `-lpython3.15t` + (`include/python3.15t`), names its JNI library `libmultiplatform_python3.15t.so` and packages + `libpython3.15t.so` and `assets//lib/python3.15t/` into the AAR for arm64-v8a and x86_64; iOS + compiles and links against the 3.15t `Python.xcframework` and gives the simulator test binary + `lib/python3.15t/`. python.org's and BeeWare's GIL mobile archives are no longer acquired. + `Status: partial` (compiled; not run on an emulator, device or simulator in this revision). - **T-2a** Free-threaded CPython on mobile through the Gradle plugin (#328 step 1; owner decision #158 of 2026-10-04, "Free-threaded is the default and the only build", baseline 3.15t; leader decision of 2026-11-13 on #328: no opt-in `pythonBindings.ios.freeThreaded` flag, 3.15t is the only build). There @@ -692,30 +709,30 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `downloadPython_ios`). Every per-slice prefix holds `lib/pythont/`: the shared `Python.xcframework/lib/pythont/` merged with `/lib-/pythont/`, with L-11's exclusions (so no `libpythont.dylib`), and the staging task fails unless `lib/pythont/os.py` is - staged. The library's own iOS klib must also be built against the 3.15t headers before a consumer app - runs on this framework (#328 step 2, done separately). Android: the plugin stages nothing for Android - (libpython and the stdlib travel inside the library's AAR, L-5); the Android archive's pin - (`android-aarch64--freethreaded`) is recorded in `python-checksums.properties`, and acquiring it - is a library-build change left to #328 steps 2 and 3. `Status: partial` (unit tests only, - `GP/IosConsumerWiringTest.kt`, `GP/IosPythonHomeLayoutTest.kt`; no simulator run, and the library's own - iOS klib is still compiled against the GIL headers). + staged. The library's own iOS klib is built against the same 3.15t framework (T-2, #328 step 2), so a + consumer never pairs a GIL klib with a 3.15t framework. Android: the plugin stages nothing for Android + (libpython and the stdlib travel inside the library's AAR, L-5); the library acquires both Android + archives itself (T-2). `Status: partial` (unit tests only, `GP/IosConsumerWiringTest.kt`, + `GP/IosPythonHomeLayoutTest.kt`; no simulator run). - **T-3** `python-multiplatform` publishes per-target, per-flavour CPython include directories as `Provider` carrying the extraction task (`cpythonIncludeDirectories` extension; see `docs/platforms/python-version-acquisition.md` §7). The default flavour of every accessor is the one - the library embeds for that target (`CPythonIncludeLayout.embeddedFlavour`: free-threaded on desktop, - GIL on Android and iOS; agent proposal, not confirmed). `Status: partial` (only the embedded flavour is extracted), + the library embeds for that target (`CPythonIncludeLayout.embeddedFlavour`: free-threaded on every + target; agent proposal, not confirmed). Android and iOS have no GIL layout. `Status: partial` (only the + embedded flavour is extracted), `GP/CPythonIncludeDirectoriesTest.kt`. - **T-4** `python-multiplatform` publishes the CPython version it embeds: extension `pythonMultiplatform`, Gradle attributes `org.thisisthepy.python.version` / `org.thisisthepy.python.free-threaded` on every consumable `*Elements` configuration, and the resource `META-INF/python-multiplatform/python.properties` in the jar and AAR (see `docs/platforms/python-version-acquisition.md` "Published version"), `GP/EmbeddedPythonVersionTest.kt`. Per T-1 the free-threaded claim is per variant: `true` on the - desktop, `macosArm64` and `linuxX64` variants and in the jar's resource, `false` on the Android, iOS, - androidNative and wasm variants and in the AAR's resource, absent on shared metadata variants; the - extension carries the desktop value (the metadata and extension rules are agent proposals, not + desktop, `macosArm64`, `linuxX64`, Android, androidNative and iOS variants and in the jar's and the + AAR's resources, `false` on the wasm variants, absent on shared metadata variants; the extension + carries the free-threaded value (the metadata and extension rules are agent proposals, not confirmed). `Status: partial`, `GP/EmbeddedPythonVersionTest.kt` - (`freeThreadedIsDecidedPerVariant`), `PM/desktopTest/.../PublishedPythonVersionResourceTest.kt`; no - test reads the AAR's resource. + (`freeThreadedIsDecidedPerVariant`), `PM/desktopTest/.../PublishedPythonVersionResourceTest.kt`, + `PM/androidInstrumentedTest/.../AndroidFreeThreadedBuildTest.kt` (`theAarSaysItIsFreeThreaded`, not + yet run on an emulator). - **T-5** Every acquired CPython reports the pinned version: `PY_VERSION` in each target's `patchlevel.h` equals `pythonVersion`, and a `downloadPython_*` task fails the build, naming the target and both versions, when it does not (issue #47: iOS shipped 3.14.6 under `pythonVersion=3.14.7`). The iOS @@ -735,8 +752,9 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. ## 9. Planned -- **N-1** Sub-interpreter-free parallelism on free-threaded CPython on every platform (today desktop only, - 3.15t by default since #158; Android and iOS under #328; see T-1, T-2 and ROADMAP §9). `Status: partial`. +- **N-1** Sub-interpreter-free parallelism on free-threaded CPython on every platform (3.15t only on + desktop since #158, and on Android, androidNative and iOS since #328, compiled but not yet run on an + emulator or simulator; wasm under #175; see T-1, T-2 and ROADMAP §9). `Status: partial`. - **N-2** Compose through Python on Android, iOS and wasm. `Status: planned`. - **N-3** Binding Kotlin/Native klib declarations at run time (B-2). `Status: planned`. - **N-4** Linux and Windows desktop runs in CI (P-2). `Status: planned`. diff --git a/docs/design/threading-and-abi.md b/docs/design/threading-and-abi.md index 71f37691c..b29e7056a 100644 --- a/docs/design/threading-and-abi.md +++ b/docs/design/threading-and-abi.md @@ -13,7 +13,7 @@ the build differs by platform (SPEC T-1): | Platform | Interpreter | Threading behaviour | |---|---|---| | Desktop JVM, `macosArm64`, `linuxX64` | `python3.15t` (free-threaded), the only desktop build | Each Kotlin thread talks to its own free-running Python thread | -| Android (ART, androidNative), iOS | `python3.15` (GIL) until #328 | Kotlin threads each attach a thread state and serialise on the GIL | +| Android (ART, androidNative; arm64-v8a and x86_64), iOS | `python3.15t` (free-threaded), the project's own build, the only mobile build since #328 | Each Kotlin thread talks to its own free-running Python thread (not yet run on an emulator or simulator) | | wasmJs | GIL until #175 | single-threaded | The per-platform fact is `embedsFreeThreadedPython` (commonMain `EmbeddedPythonBuild.kt`, an @@ -99,10 +99,10 @@ Design for two artefacts, but keep the split shallow enough to collapse later. Desktop (macOS arm64/x86_64, Linux x86_64, Windows x86_64) has GIL and free-threaded builds on the python-build-standalone releases; the build uses only the `-freethreaded` archive (#158). -**Android (python.org) and iOS (Python-Apple-support) publish no free-threaded build**, so those -platforms stay GIL until the project's own builds land (#163, #328). Until then the threading model, -and the object-lifetime and thread-state design layered on top, are split across platforms; that is -the accepted cost of moving desktop first. The maintainer accepted the wheel-coverage cost on #158 +**Android (python.org) and iOS (Python-Apple-support) publish no free-threaded build**, so the project +builds its own (#163, release `cpython-3.15.0t-mobile`), and since #328 the library embeds those on +Android, androidNative and iOS. Only wasm still has a GIL build (#175), so the threading model is split +between wasm and everything else. The maintainer accepted the wheel-coverage cost on #158 ("third-party cp315t / abi3t wheels lag the release for a while; pypackpack builds from sdist where they are missing"). Table as of the 20260807 release, kept in the archive note above. diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md index b53e50ba1..8d63c90b2 100644 --- a/docs/locale/README_ko.md +++ b/docs/locale/README_ko.md @@ -196,7 +196,7 @@ Python → Kotlin 바인딩 코드가 없다. 자세한 내용: [설계 문서]( | 빌드된 jar 바인딩, Python 에서 Compose | ✅ 데스크톱 · ⏳ Android / iOS / wasm | | Kotlin/Native klib 바인딩 | 🟡 스캐너만 | | `.pyi` 스텁 생성 | 🟡 부분 | -| free-threaded CPython 3.15t (#158 이후 데스크톱의 유일한 빌드; Android / iOS 는 #328 까지 GIL) | 🟡 데스크톱 · ⏳ Android / iOS / wasm | +| free-threaded CPython 3.15t (데스크톱은 #158, Android / iOS 는 #328 이후 유일한 빌드; wasm 은 #175 까지 GIL) | 🟡 데스크톱 · 🟡 Android / iOS (컴파일만 확인, 에뮬레이터·시뮬레이터 실행 전) · ⏳ wasm | ## 🧱 생태계 diff --git a/docs/platforms/free-threaded-mobile.md b/docs/platforms/free-threaded-mobile.md index 8146231ae..a276cbc6c 100644 --- a/docs/platforms/free-threaded-mobile.md +++ b/docs/platforms/free-threaded-mobile.md @@ -1,19 +1,24 @@ # Free-threaded CPython for Android and iOS -Status: build recipe written from the CPython 3.15.0 source; **not yet run in CI, -not yet run on a device or emulator.** Tracking issue: #163. +Status: built in CI and published as the GitHub release `cpython-3.15.0t-mobile` +(aarch64 Android run 39013371415, x86_64 Android run 43300549614 (#396), iOS run +39014667720); the SHA-256 of each archive is in the release's `SHA256SUMS` and pinned in +`python-checksums.properties`. Since #328 step 2 the library embeds these builds on Android +(arm64-v8a and x86_64), androidNative and iOS. **Not yet run on a device, emulator or +simulator.** Tracking issues: #163, #328. python-multiplatform gets its parallelism from free-threaded CPython (decision 8 in -`AGENTS.md`). Desktop uses the python.org 3.14t builds. No prebuilt 3.15t exists for -Android or iOS, so the workflow `.github/workflows/build-ft-mobile.yml` builds it with +`AGENTS.md`). Desktop uses python-build-standalone's 3.15t builds. No prebuilt 3.15t exists +for Android or iOS, so the workflow `.github/workflows/build-ft-mobile.yml` builds it with CPython's own mobile tooling. ## What is built | Platform | Host | Tooling | Output | |---|---|---|---| -| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0-aarch64-linux-android.tar.gz` | -| iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0-iOS-XCframework.tar.gz` | +| Android | `aarch64-linux-android` | `Platforms/Android` | `python-3.15.0t-aarch64-linux-android.tar.gz` | +| Android | `x86_64-linux-android` (#396) | `Platforms/Android` | `python-3.15.0t-x86_64-linux-android.tar.gz` | +| iOS | `arm64-apple-ios` (device), `arm64-apple-ios-simulator`, `x86_64-apple-ios-simulator` | `Platforms/Apple` | `python-3.15.0t-iOS-XCframework.tar.gz` | The iOS tooling only builds an XCframework from all of its slices, so the simulator hosts are built as well as the device. There is no cheaper arm64-only path in the @@ -105,16 +110,32 @@ The sha256 of each archive is written to the job summary and the archives are up as workflow artifacts (`python-t-android-aarch64`, `python-t-ios-xcframework`). +## How the library uses them (#328 step 2) + +- `python-multiplatform/build.gradle.kts` downloads the three archives from the release + (`downloadPython_android_aarch64`, `downloadPython_android_x86_64`, `downloadPython_ios`), + checks each against `python-checksums.properties` (`android-aarch64--freethreaded`, + `android-x86_64--freethreaded`, `ios--freethreaded`) and checks the unpacked headers: + `PY_VERSION` equals `pythonVersion`, and every `pyconfig` header defines + `Py_GIL_DISABLED 1` (the iOS `pyconfig.h` wrapper is skipped in favour of the + `pyconfig-.h` it includes). +- Android unpacks under `extracted//android--freethreaded/`, links androidNative + with `-lpython3.15t` against `include/python3.15t`, names the JNI library + `libmultiplatform_python3.15t.so` (what `manager.loadLibPython` asks for), and packages + `libpython3.15t.so` and `assets//lib/python3.15t/` into the AAR for arm64-v8a and x86_64. +- iOS compiles and links against the 3.15t `Python.xcframework`; the simulator test binary + gets `lib/python3.15t/` from `Python.xcframework/lib/python3.15t` and the slice's + `lib-arm64/python3.15t`. The plugin's iOS consumer wiring stages the same framework (SPEC T-2a). +- `embedsFreeThreadedPython` is `true` in `androidMain`, `artMain` and `iosMain`, so + `Versions.abiFlags` is `t` and the stdlib, stamp and library names follow. + ## Not verified -- The workflow has not run yet. Whether 3.15.0 configures and compiles with - `--disable-gil` for these targets is unknown; free-threaded support on Android and iOS - is not a documented CPython target. - Nothing has been loaded on a device, emulator or simulator. No import of `_ssl`, - `_ctypes` or extension modules, no GIL-off test run, no thread-safety check. + `_ctypes` or extension modules, no GIL-off test run, no thread-safety check. The tests + that would show it are `AndroidFreeThreadedBuildTest` (instrumented), + `AndroidNativeFreeThreadedBuildTest` and `IosFreeThreadedBuildTest`; they are compiled, + not run. - Whether the prebuilt dependency tarballs and the Xcode on `macos-15` suit 3.15.0 is - unconfirmed. -- Integration with the Gradle plugin's `PYTHONHOME` staging, with the Android - `libpython` loading code, and with the iOS framework embedding (which expects - `Python.framework` and a non-`t` library name in places) is not done. + unconfirmed beyond the build itself. - The x86_64 simulator slice is built only because the packaging step requires it. diff --git a/docs/platforms/python-version-acquisition.md b/docs/platforms/python-version-acquisition.md index 557dd7fcb..325fa57b5 100644 --- a/docs/platforms/python-version-acquisition.md +++ b/docs/platforms/python-version-acquisition.md @@ -28,16 +28,31 @@ The project currently uses `python-build-standalone` (maintained by Astral). For leader 2026-11-13: "flip the desktop default to 3.15t now"). Every desktop `downloadPython_*` task fetches `...-freethreaded-install_only_stripped.tar.gz` and unpacks it under `/-freethreaded/`. The `-PpythonFreeThreaded` property that used to choose between the two is removed; passing it fails the - build. Android and iOS (section 3) are still GIL until #328. + build. Android and iOS (section 3) embed the project's own 3.15t builds since #328. ## 3. Android and iOS Support -* **Android:** Prebuilt artifacts for Android are provided natively by python.org. These archives are plain NDK cross-compilations (e.g., CPython's own build tree). They can be found at `https://www.python.org/ftp/python//python---linux-android.tar.gz`. -* **iOS (3.14):** pinned to BeeWare `3.14-b11` (Python 3.14.7). `3.14-b10`, pinned earlier, shipped 3.14.6 while every other target shipped 3.14.7 (issue #47); b11 is the first 3.14 build with 3.14.7 (release 2026-09-04; its `VERSIONS` file and all four `patchlevel.h` say 3.14.7). The build now checks `PY_VERSION` in the extracted headers against `pythonVersion` for every target and fails if they differ. python.org's own iOS XCframework starts at 3.15 (section 5), so there is no alternative 3.14 source. -* **iOS:** Prebuilt artifacts for iOS are provided by the BeeWare project via their `Python-Apple-support` repository, which are CPython's own iOS build layouts (e.g. `prefix: iOS/Frameworks/...`). They are located at `https://github.com/beeware/Python-Apple-support/releases`. The kivy toolchains are **not** the source of these artifacts. -* **python-build-standalone Status:** `python-build-standalone` has moved from indygreg to Astral (`https://github.com/astral-sh/python-build-standalone`). It **does not** publish prebuilt binaries for `android` or `ios`. +* **Acquired: the project's own free-threaded 3.15t builds, and only those** (#328 step 2; maintainer on + #158, 2026-10-04: "Free-threaded is the default and the only build. The GIL build is not kept as an + option"; leader decision 2026-11-13 on #328: no opt-in flag on mobile; the owner approved the x86_64 + Android 3.15t build the same day, so arm64-v8a and x86_64 both stay supported). They are built by + `.github/workflows/build-ft-mobile.yml` (#163, `free-threaded-mobile.md`) and published on the GitHub + release `cpython-t-mobile` of `thisisthepy/python-multiplatform`: + `python-t-aarch64-linux-android.tar.gz`, `python-t-x86_64-linux-android.tar.gz` and + `python-t-iOS-XCframework.tar.gz`, at + `https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-t-mobile/`. +* **Layout.** Android: `prefix/lib/libpython3.15t.so`, `prefix/include/python3.15t/`, + `prefix/lib/python3.15t/` (with `config-3.15t--linux-android/`), unpacked under + `/android--freethreaded/`. iOS: `Python.xcframework//Python.framework/Headers` + (`pyconfig.h` wraps `pyconfig-.h`), `Python.xcframework/lib/python3.15t/` and + `/lib-/python3.15t/`, unpacked under `/ios/`. +* **Checks.** The lockfile pin (section 5), `PY_VERSION` against `pythonVersion` (issue #47), and + `Py_GIL_DISABLED 1` in every `pyconfig` header of the unpacked tree. +* **Not acquired any more:** python.org's GIL Android archives and GIL iOS XCframework (3.15), and BeeWare's + `Python-Apple-support` (3.14 and earlier, GIL). Neither publishes a free-threaded mobile build, and + the GIL build is not kept (#158). Their pins and the `pythonAppleSupportBuild` property are removed. +* **python-build-standalone Status:** `python-build-standalone` has moved from indygreg to Astral (`https://github.com/astral-sh/python-build-standalone`). It **does not** publish prebuilt binaries for `android` or `ios`. * **Prebuilt-cpython Status:** The `python/prebuilt-cpython` repository is currently a planning repository containing no artifacts, so it is not a source. -* **Conclusion:** We acquire Android artifacts directly from python.org, and iOS artifacts from BeeWare's Python-Apple-support, eliminating the need for kivy toolchain dependency. ## 4. Stable ABI and free-threaded builds @@ -50,22 +65,27 @@ The project currently uses `python-build-standalone` (maintained by Astral). For due to opaque structural changes", "wait for 3.15t") and was wrong; see [`../design/threading-and-abi.md`](../design/threading-and-abi.md) and [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). -* **Scope:** free-threaded prebuilts exist only for the desktop targets (section 2); Android (python.org) - and iOS (Python-Apple-support) publish none, so desktop is free-threaded and Android and iOS stay GIL - until the project's own builds land (#163, #328). Which build each platform embeds is the library's - per-platform `embedsFreeThreadedPython` (SPEC T-1). The interpreter version is `pythonVersion` in +* **Scope:** free-threaded prebuilts from third parties exist only for the desktop targets (section 2); + Android and iOS use the project's own 3.15t builds (section 3, #163, #328), and wasm stays GIL until + #175. Which build each platform embeds is the library's per-platform `embedsFreeThreadedPython` + (SPEC T-1). The interpreter version is `pythonVersion` in `gradle.properties`. ## 5. Security and Integrity Verification * **Desktop:** `python-build-standalone` provides a `SHA256SUMS` file with every release. The build script verifies the downloaded archive against this manifest. -* **Android (python.org) and iOS (python.org, 3.15+), Sigstore verified.** This bullet used to say that "full Sigstore verification directly inside Gradle is unreasonable without shelling out to external tools (like `sigstore-python` or the `cosign` CLI)". **That was wrong**, and it is now implemented. `dev.sigstore:sigstore-java` does the whole verification in-process, certificate chain, Rekor inclusion and OIDC identity, with no external binary. python.org publishes a sibling `.sigstore` bundle for both Android tarballs and, from 3.15, the iOS XCframework. (The extension is `.sigstore`; `.sigstore.json` returns 404.) There is still no plain checksum manifest, so the lockfile continues to pin the digest as well. -* **The signer identity is pinned, and has to be version-keyed.** Verifying a bundle *without* pinning an identity proves only that somebody holding a Sigstore certificate signed the bytes, which anyone can arrange. The build pins the Fulcio SAN and OIDC issuer of the actual CPython release manager, per . That must be a map rather than a constant, because it changes per release series: 3.14/3.15 are `hugo@python.org` via `https://github.com/login/oauth`, whereas 3.12/3.13 are `thomas@python.org` via `https://accounts.google.com`. An unrecorded series is a hard build failure, never a silent skip. -* **Opt-in, `-PverifyPythonSignatures=true`.** Off by default because `sigstore-java` pulls in grpc-netty-shaded, protobuf, bouncycastle and guava, and because fetching the TUF trust root needs network access, which would turn an offline build from working into failing. Gradle resolves the configuration lazily, so a default build downloads none of it (measured: a default `downloadPython_android_*` run mentions Sigstore zero times). +* **Android and iOS (the project's `cpython-t-mobile` release): the lockfile pin.** The release + carries a `SHA256SUMS` and no Sigstore bundle; the pins `android-aarch64--freethreaded`, + `android-x86_64--freethreaded` and `ios--freethreaded` were checked against it and against a + local `shasum` when they were written (2026-11-13). The build checks the lockfile only. +* **python.org's Android and iOS archives used to be Sigstore verified** through their sibling + `.sigstore` bundles, with the release manager's identity pinned per series. Those archives are + GIL builds and are no longer acquired (section 3), so that verification path and its identity map + were removed from the build. The record is in git history. +* **Opt-in, `-PverifyPythonSignatures=true`.** Off by default because `sigstore-java` pulls in grpc-netty-shaded, protobuf, bouncycastle and guava, and because fetching the TUF trust root needs network access, which would turn an offline build from working into failing. Gradle resolves the configuration lazily, so a default build downloads none of it. * **Desktop (`python-build-standalone`), Sigstore verified via GitHub Attestations.** The release carries 853 assets and the only non-archive among them is `SHA256SUMS`: there is no `.sigstore` file. Provenance is published through GitHub's *attestations* API instead, keyed by artifact **digest** rather than filename, and that endpoint is rate-limited to 60 requests/hour unauthenticated. Because of this rate limit and the bundle size (~212KB each), the build does not check these bundles into the repository. Instead, when `-PverifyPythonSignatures=true` is set, it fetches the DSSE bundle from the API, caches it locally as `.sigstore`, and uses `sigstore-java` to verify it against the pinned `astral-sh/python-build-standalone` workflow identity. Desktop is therefore protected by the lockfile, the release's own `SHA256SUMS`, and Sigstore. -* **iOS (BeeWare, ≤ 3.14; `3.14-b11` pin `b591f330...ab7b`, taken from GitHub's asset digest and recomputed locally), nothing exists to verify against.** The `Python-Apple-support` releases publish five `tar.gz` assets and nothing else: no checksums, no signatures, and no GitHub attestations. The lockfile pin is the only honest instrument available here, and no amount of build wiring changes that. * **The lockfile is not superseded by any of this.** The two gates prove different things, the lockfile says "these are the exact bytes this repository reviewed and pinned", Sigstore says "these are the bytes the release manager actually signed". The lockfile is also the only check that works offline and the only one covering every source, so it stays unconditional and Sigstore is layered on top of it. -* **The verification is known to fail when it should**, which is the only thing that makes it worth having. Two negative controls were run against the real 3.14.7 Android bundle: flipping one base64 character in `messageSignature.signature` gave `KeylessVerificationException: Artifact signature was not valid`, and pointing the identity map at the 3.13 release manager gave `No provided certificate identities matched values in certificate`. Both failed the build with a non-zero exit. +* **The verification is known to fail when it should**, which is the only thing that makes it worth having. Two negative controls were run against the real 3.14.7 Android bundle (python.org, before #328 removed that source): flipping one base64 character in `messageSignature.signature` gave `KeylessVerificationException: Artifact signature was not valid`, and pointing the identity map at the 3.13 release manager gave `No provided certificate identities matched values in certificate`. Both failed the build with a non-zero exit. ## 6. WebAssembly @@ -86,17 +106,19 @@ Compiling a C extension against the embedded CPython needs that CPython's header `windows-x86_64`, `android-aarch64`, `android-x86_64`, `ios-arm64`, `ios-arm64_x86_64-simulator` (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.FREE_THREADED` or `GIL`; the default is the one the library embeds for that target, `CPythonIncludeLayout.embeddedFlavour(target)`: - `FREE_THREADED` on desktop (#158), `GIL` on Android and iOS. (Making the default per target, rather - than `GIL` everywhere, is an agent proposal, not confirmed: with `GIL` the documented - `includeDir("macos-aarch64")` would throw on every desktop build.) + `FREE_THREADED` on every target (desktop #158, Android and iOS #328). Android and iOS have no `GIL` + layout at all. (Making the default follow the embedded build, rather than `GIL` everywhere, is an + agent proposal, not confirmed: with `GIL` the documented `includeDir("macos-aarch64")` would throw + on every build.) * **Value:** `Provider` of the directory that directly contains `Python.h`, i.e. the `-I` directory: `/[-freethreaded]/python/include/python3.14[t]` on desktop (Windows has no - `python3.14` level), `/android-*/prefix/include/python3.14`, - `/ios/Python.xcframework//include/python3.14`. + `python3.14` level), `/android-*-freethreaded/prefix/include/python3.15t`, + `/ios/Python.xcframework//Python.framework/Headers`. * **Task dependency:** the provider carries the matching `downloadPython_*` task; using it as a task input makes Gradle download and extract first. -* **Only the embedded flavour is extracted.** Desktop extracts the free-threaded tree only; asking for the - other flavour throws a `GradleException` naming #158 (`isAvailable` tests it without throwing). +* **Only the embedded flavour is extracted.** Every target extracts the free-threaded tree only; asking for + the other flavour throws a `GradleException` naming #158 (desktop) or #328 (Android, iOS); + `isAvailable` tests it without throwing. ```kotlin // consumer build.gradle.kts @@ -117,12 +139,12 @@ The same extension exposes the library to link against (issue #56): | Target | Link library (under `/`) | |---|---| -| `android-aarch64`, `android-x86_64` | `/prefix/lib/libpython3.14.so` | +| `android-aarch64`, `android-x86_64` | `-freethreaded/prefix/lib/libpython3.15t.so` | | `windows-x86_64` | `windows-x86_64/python/libs/python314.lib` (`python3.lib` is the stable-ABI one) | | `windows-x86_64` free-threaded | `windows-x86_64-freethreaded/python/libs/python314t.lib` | | macOS, Linux, iOS | none -- `isLinkRequired(target)` is false; `libraryDir`/`libraryFile` throw `GradleException` | -Android has no free-threaded build, so that combination throws. On Windows the default is the +Android has no GIL build since #328, so that combination throws. On Windows the default is the free-threaded `python315t.lib`; the GIL one is not extracted. `isLinkAvailable(target, flavour)` tests without throwing; the other flavour fails naming #158, as for `includeDir`. @@ -130,22 +152,22 @@ tests without throwing; the other flavour fails naming #158, as for `includeDir` A consumer that builds against `python-multiplatform` must know which CPython it embeds (issue #61). The answer is published three ways. The version is `pythonVersion` in `gradle.properties`; whether -the build is free-threaded differs by platform (#158: desktop 3.15t, Android and iOS GIL 3.15 until -#328, wasm GIL until #175), so each published artefact states its own. +the build is free-threaded differs by platform (desktop 3.15t since #158, Android, androidNative and +iOS 3.15t since #328, wasm GIL until #175), so each published artefact states its own. * **Extension** `pythonMultiplatform` (`python.multiplatform.gradle.EmbeddedPythonVersion`): `pythonVersion` (`3.15.0`), `majorMinor` (`3.15`), `freeThreaded`. One value per project: the - desktop one (`true`; agent proposal, not confirmed). For any other platform, read its variant's + free-threaded one every non-wasm platform embeds (`true`; agent proposal, not confirmed). For any other platform, read its variant's attribute or its resource. * **Gradle attributes** on every consumable `*Elements` configuration (so they land in the published module metadata): `org.thisisthepy.python.version` and `org.thisisthepy.python.free-threaded` (both `String`; the latter is `"true"`/`"false"`). The free-threaded one is per variant: `"true"` on - `desktop*`, `macosArm64*` and `linuxX64*` variants, `"false"` on the Android, iOS, androidNative and - wasm variants, and absent on shared metadata variants, which carry no CPython of their own (the + `desktop*`, `macosArm64*`, `linuxX64*`, `android*` (the AAR and androidNative) and `ios*` variants, + `"false"` on the wasm variants, and absent on shared metadata variants, which carry no CPython of their own (the metadata rule is an agent proposal, not confirmed). * **Resource** `META-INF/python-multiplatform/python.properties` (`pythonVersion=`, `freeThreaded=`), inside the desktop jar (`freeThreaded=true`, from `generateDesktopPythonProperties`) and the Android - AAR's `classes.jar` (`freeThreaded=false`, from `generateAndroidPythonProperties`). + AAR's `classes.jar` (`freeThreaded=true` since #328, from `generateAndroidPythonProperties`). ```kotlin // composite build (includeBuild): read the extension diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index 547a8f162..e4d11b332 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -164,8 +164,9 @@ Desktop (JVM, `macosArm64`, `linuxX64`) embeds the free-threaded CPython 3.15t a (maintainer, #158, 2026-10-04: "Free-threaded is the default and the only build. The GIL build is not kept as an option. Baseline is CPython 3.15t"; leader, 2026-11-13: "flip the desktop default to 3.15t now, before the #377 alpha03 release"). `-PpythonFreeThreaded` is removed; the build fails if it is -passed. Android and iOS still embed the GIL 3.15 (9.2), wasm GIL (#175); the per-platform fact is -`embedsFreeThreadedPython` (SPEC T-1). The 3.15t default has not yet been observed running the desktop +passed. Android (arm64-v8a, x86_64), androidNative and iOS embed the project's own 3.15t with no +opt-in flag (#328 step 2; leader 2026-11-13), compiled but not yet run on an emulator, device or +simulator (9.2); wasm is GIL (#175); the per-platform fact is `embedsFreeThreadedPython` (SPEC T-1). The 3.15t default has not yet been observed running the desktop suite; the last recorded free-threaded run is 3.14t as an opt-in: 236 tests, 0 failures, 1 skipped, the same count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is drained by the eval-loop checkpoint (`PM/commonTest/.../ref/EvalCheckpointTest.kt`, @@ -175,19 +176,23 @@ The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done free-threaded ones, are pinned in `python-checksums.properties`. - **9.1 `autoDrainInterval`'s default on the GIL build, open (decision).** Today - `if (embedsFreeThreadedPython) 32 else 0` (`Python3.kt`), so on by default on desktop and off on - Android and iOS until they move (9.2); once every platform is free-threaded this question closes. Off means a pure C-API embedder never + `if (embedsFreeThreadedPython) 32 else 0` (`Python3.kt`), so on by default on desktop, Android and + iOS (#328) and off only on wasm until #175; once wasm is free-threaded this question closes. Off means a pure C-API embedder never runs the cyclic collector: 20 000+ cyclic objects left with the automatic path off against ~1 970 with it on (`docs/investigations/gc-scheduling-investigation.md` §6–§7). On means a `__del__` or weakref callback can run at a `withGIL` exit the caller did not ask to yield from. The checkpoint costs ~128 ns over a 154.6 ns `withGIL` floor, ~4 ns amortised (`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`). A one-line change once decided. -- **9.2 Free-threading beyond desktop, open (#328, wasm #175).** No free-threaded CPython prebuilt +- **9.2 Free-threading beyond desktop, open (#328 run, wasm #175).** No free-threaded CPython prebuilt exists for Android (python.org ships `libpython3.14.so`/`3.15.so` only) or iOS (neither BeeWare nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12; the project builds its own (#163, - `.github/workflows/build-ft-mobile.yml`). Moving a platform means flipping its - `embedsFreeThreadedPython` actual together with its acquisition, its `python.properties` resource - and its variant attributes. SPEC N-1. + `.github/workflows/build-ft-mobile.yml`, release `cpython-3.15.0t-mobile`). #328 step 2 moved the + library: Android (arm64-v8a and x86_64), androidNative and iOS acquire those archives, link + `libpython3.15t`, package `lib/python3.15t`, report `abiFlags = "t"`, and publish + `freeThreaded=true`; the python.org and BeeWare GIL mobile archives and their pins are gone. Left: + an Android emulator run (`AndroidFreeThreadedBuildTest`, the instrumented suite, the androidNative + suite) and an iOS simulator run (`IosFreeThreadedBuildTest`, the ios-consumer fixture, sample DEMO + 1/8) with `Py_GIL_DISABLED` observed at run time; then wasm (#175). SPEC N-1. ## 10. WASM diff --git a/gradle.properties b/gradle.properties index c800b0ec5..8af7874d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -29,7 +29,6 @@ org.jetbrains.compose.hot.reload.disable=true # Python Build pythonVersion=3.15.0 pythonBuildStandaloneRelease=20261013 -pythonAppleSupportBuild=b11 # Every published artifact (library, KSP processor, Gradle plugin) is versioned # "$pythonVersion-$libraryVersionSuffix"; each build derives it from these two values and fails if # its own version disagrees (#70). diff --git a/python-checksums.properties b/python-checksums.properties index b3899bd58..956b5db07 100644 --- a/python-checksums.properties +++ b/python-checksums.properties @@ -1,12 +1,7 @@ # Python Multiplatform Checksums -android-aarch64-3.14.7=6d50cc3aa66e414a439594089bcdfb5f1264358155c70c1f00471c24cfb477fb -android-aarch64-3.15.0=571e4cf4e0aee3d1ab54a52ca3d4e6845b85ec0bb327f7debfb873172f445dab android-aarch64-3.15.0-freethreaded=40995c530c3e454d829b4fdea8404d8c0b0e58f60c2d41d400daa1fb0ab65991 -android-x86_64-3.14.7=2c16ce2359565cd8c24f86cfb75630768ba6607e732946b294b969797f583b60 -android-x86_64-3.15.0=2413d51948d612cf42baefba25295bb83c2ed84df5726af6fe5bde2dec5d87e7 -ios-3.14-b11=b591f3301bd22a4f423c49c746cac9e55558b909fd14d6eb8327ccc62234ab7b +android-x86_64-3.15.0-freethreaded=d52580653858c429f542db0ca4ecfe12452faf6c9a967bb4b62797df7c2c8d24 ios-3.15.0-freethreaded=b8c0a4b32d31738f6ead7ace28acee0cb2a590bd0ca7015e9c32af613445e6b5 -ios-3.15.0-pythonorg=68e98ab35cc0643ba3a227a8c7768da45c2a5e67dbe1bc26598537e96680fd19 linux-x86_64-3.14.7-20260807=a2478d654ed51d443bae21ec20ad927f116b4f5aae4094ab74918a6aa38f0575 linux-x86_64-3.14.7-20260807-freethreaded=13f80b7fc242814c0be4e2a4d8283f39c7e1c0f3f0d8f6d6a15cedb5b62eb6d3 linux-x86_64-3.15.0-20261013=69740dcd9b3480b3d53760f2d3b75582adb14fa54434628dea32480ddede542a diff --git a/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts b/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts index a5d1c44db..205f49cf5 100644 --- a/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts +++ b/python-multiplatform-gradle-plugin/fixtures/klib-artifact/build.gradle.kts @@ -52,9 +52,10 @@ kotlin { // Mirrors `:python-multiplatform-ksp:fixtures:app`'s own androidNativeArm64 linker config: the test binary needs // the same CPython symbols resolved at final link time, since `python-multiplatform`'s klib // only declares them via cinterop and does not itself embed a static libpython. - val targetExtractDir = "$pythonDownloadDir/extracted/$fixturePythonVersion/android-aarch64/prefix" + // The free-threaded 3.15t tree, the only Android build (#328). + val targetExtractDir = "$pythonDownloadDir/extracted/$fixturePythonVersion/android-aarch64-freethreaded/prefix" binaries.getTest(org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG).linkerOpts.addAll( - listOf("-L$targetExtractDir/lib/", "-lpython$fixturePythonLibVersion", "-Wl,--allow-shlib-undefined"), + listOf("-L$targetExtractDir/lib/", "-lpython${fixturePythonLibVersion}t", "-Wl,--allow-shlib-undefined"), ) } // Issue #234: the simulator target, so the suite runs through Gradle (`iosSimulatorArm64Test`) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt index a95951aa5..5f438c990 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/AcquiredHeaderVersion.kt @@ -14,9 +14,11 @@ object AcquiredHeaderVersion { requireNotNull(pyVersion.find(patchlevelText)) { "no PY_VERSION in patchlevel.h" }.groupValues[1] fun patchlevelFile(extractedRoot: File, target: String, pythonVersion: String): File = - extractedRoot.resolve(CPythonIncludeLayout.relativePath(target, CPythonFlavour.GIL, pythonVersion)).resolve("patchlevel.h") + extractedRoot.resolve( + CPythonIncludeLayout.relativePath(target, CPythonIncludeLayout.embeddedFlavour(target), pythonVersion), + ).resolve("patchlevel.h") - /** target -> description, for every GIL target whose headers exist and report a different version. */ + /** target -> description, for every target whose embedded headers exist and report a different version. */ fun mismatches(extractedRoot: File, pythonVersion: String): Map = CPythonIncludeLayout.allTargets.mapNotNull { t -> val f = patchlevelFile(extractedRoot, t, pythonVersion) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt index f5c87597e..36b0beb1d 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt @@ -30,14 +30,19 @@ object CPythonIncludeLayout { /** * The flavour `python-multiplatform` embeds for [target], which is the only one its build - * extracts: free-threaded on desktop (issue #158, maintainer 2026-10-04: "Free-threaded is the - * default and the only build"), GIL on Android and iOS until #328. + * extracts: free-threaded on every target (issue #158, maintainer 2026-10-04: "Free-threaded is + * the default and the only build"; desktop since #398, Android and iOS since #328). */ - fun embeddedFlavour(target: String): CPythonFlavour = - if (target in desktopTargets) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL + @Suppress("UNUSED_PARAMETER") + fun embeddedFlavour(target: String): CPythonFlavour = CPythonFlavour.FREE_THREADED + /** + * Whether a layout exists for [target] in [flavour]. Free-threaded exists everywhere. A GIL + * layout is kept for the desktop targets only (python-build-standalone publishes both, and the + * paths differ); Android and iOS have no GIL build to lay out since #328. + */ fun supports(target: String, flavour: CPythonFlavour): Boolean = - target in allTargets && (flavour == CPythonFlavour.GIL || target in desktopTargets) + target in allTargets && (flavour == CPythonFlavour.FREE_THREADED || target in desktopTargets) /** The extraction task that produces [target] -- the name `python-multiplatform/build.gradle.kts` registers. */ fun downloadTaskName(target: String): String = when (target) { @@ -60,9 +65,11 @@ object CPythonIncludeLayout { val dir = if (flavour == CPythonFlavour.FREE_THREADED) "$target-freethreaded" else target "$dir/python/include/python$tag" } - in androidTargets -> "$target/prefix/include/python$tag" + // `-freethreaded` in the directory name, as on desktop: a GIL `android-aarch64/` tree from + // an older checkout must never be mistaken for the 3.15t one (#328). + in androidTargets -> "$target-freethreaded/prefix/include/python$tag" // The framework's own Headers, which is also what the library's iOS cinterop reads: the - // 3.15 iOS support archive no longer ships `/include/python3.15` beside it (#313). + // iOS support archive ships no `/include/python3.15t` beside it (#313). else -> "ios/Python.xcframework/$target/Python.framework/Headers" } } @@ -72,9 +79,12 @@ object CPythonIncludeLayout { /** Android links `libpython3.x.so`, Windows links the import library; macOS/Linux/iOS need none. */ fun isLinkRequired(target: String): Boolean = target in androidTargets || target == "windows-x86_64" - /** Free-threaded link libraries exist on Windows only (`python314t.lib`); Android has no free-threaded build. */ + /** + * Android links `libpython3.15t.so` (free-threaded only since #328); Windows has both + * `python315.lib` and `python315t.lib`. + */ fun supportsLinkLibrary(target: String, flavour: CPythonFlavour): Boolean = - isLinkRequired(target) && (flavour == CPythonFlavour.GIL || target == "windows-x86_64") + isLinkRequired(target) && (flavour == CPythonFlavour.FREE_THREADED || target == "windows-x86_64") /** Directory holding the link library, relative to the version-keyed extraction root. */ fun linkLibraryDirRelativePath(target: String, flavour: CPythonFlavour): String { @@ -83,11 +93,11 @@ object CPythonIncludeLayout { "windows-x86_64" -> (if (flavour == CPythonFlavour.FREE_THREADED) "windows-x86_64-freethreaded" else "windows-x86_64") + "/python/libs" - else -> "$target/prefix/lib" + else -> "$target-freethreaded/prefix/lib" } } - /** File name of the link library, e.g. `libpython3.14.so` or `python314.lib` (`python314t.lib` free-threaded). */ + /** File name of the link library, e.g. `libpython3.15t.so` or `python315.lib` (`python315t.lib` free-threaded). */ fun linkLibraryFileName(target: String, flavour: CPythonFlavour, pythonVersion: String): String { require(supportsLinkLibrary(target, flavour)) { "no $flavour link library for target '$target'" } val parts = pythonVersion.split('.') @@ -95,7 +105,7 @@ object CPythonIncludeLayout { return if (target == "windows-x86_64") { "python${parts[0]}${parts[1]}" + (if (flavour == CPythonFlavour.FREE_THREADED) "t" else "") + ".lib" } else { - "libpython${parts[0]}.${parts[1]}.so" + "libpython${parts[0]}.${parts[1]}t.so" } } } @@ -106,8 +116,8 @@ object CPythonIncludeLayout { * triggers the download and unpack. * * Only the flavour the library embeds ([CPythonIncludeLayout.embeddedFlavour]: free-threaded on - * desktop, GIL on Android and iOS) has an extraction task, and it is the default `flavour`; asking - * for the other one fails loudly rather than returning a path nothing will ever create. + * every target since #328) has an extraction task, and it is the default `flavour`; asking for the + * other one fails loudly rather than returning a path nothing will ever create. * * @param extractedRoot the version-keyed extraction directory * @param extractionTask the task that fills [extractedRoot] for a target+flavour, or null when this build does not extract it @@ -128,12 +138,12 @@ class CPythonIncludeDirectories( if (!CPythonIncludeLayout.supports(target, flavour)) { throw GradleException( "No $flavour CPython include directory for '$target'. Targets: ${CPythonIncludeLayout.allTargets}; " + - "free-threaded exists for desktop targets only.", + "Android and iOS have no GIL build (#328).", ) } val task = extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour the library embeds is downloaded: free-threaded on desktop (#158), GIL on Android and iOS.", + "Only the flavour the library embeds is downloaded: free-threaded on every target (#158, #328).", ) val relative = CPythonIncludeLayout.relativePath(target, flavour, pythonVersion) // `TaskProvider.map` keeps the task as the producer, so the dependency travels with the value. @@ -151,12 +161,12 @@ class CPythonIncludeDirectories( if (!CPythonIncludeLayout.supportsLinkLibrary(target, flavour)) { throw GradleException( "No $flavour CPython link library for '$target'. Only android-* and windows-x86_64 link against " + - "CPython; macOS, Linux and iOS need none (isLinkRequired=false); free-threaded exists on Windows only.", + "CPython; macOS, Linux and iOS need none (isLinkRequired=false); Android has no GIL build (#328).", ) } return extractionTask(target, flavour) ?: throw GradleException( "The $flavour CPython for '$target' is not extracted by this build. " + - "Only the flavour the library embeds is downloaded: free-threaded on desktop (#158), GIL on Android and iOS.", + "Only the flavour the library embeds is downloaded: free-threaded on every target (#158, #328).", ) } @@ -167,7 +177,7 @@ class CPythonIncludeDirectories( return task.map { extractedRoot.dir(relative) } } - /** The link library file itself (`libpython3.14.so` / `python314.lib`); carries the extraction task. */ + /** The link library file itself (`libpython3.15t.so` / `python315t.lib`); carries the extraction task. */ fun libraryFile(target: String, flavour: CPythonFlavour = CPythonIncludeLayout.embeddedFlavour(target)): Provider { val task = linkTask(target, flavour) val relative = CPythonIncludeLayout.linkLibraryDirRelativePath(target, flavour) + "/" + diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt index d1bf42786..11033e031 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/EmbeddedPythonVersion.kt @@ -12,7 +12,7 @@ import org.gradle.api.Project * - as the classpath resource [RESOURCE_PATH] (read from the jar/AAR, no Gradle needed). * * One instance describes one build of CPython. A project whose platforms embed different builds - * (issue #158: desktop 3.15t, Android and iOS the GIL 3.15 until #328) passes a per-variant + * (issues #158, #328: desktop, Android and iOS 3.15t, wasm a GIL build until #175) passes a per-variant * `freeThreadedFor` to [publishAttributes] / [register], so each published variant states the build * it really carries, and renders one resource per artefact. */ diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt index 311cf9f24..cfb17b23e 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/IosConsumerWiring.kt @@ -48,7 +48,7 @@ internal object FreeThreadedMobileRelease { fun iosArchiveName(pythonVersion: String): String = "python-${pythonVersion}t-iOS-XCframework.tar.gz" - /** `aarch64` is the only Android architecture the release carries (no x86_64 build exists). */ + /** `aarch64` and `x86_64` (#396; the owner approved the x86_64 3.15t build on 2026-11-13). The library build reads them. */ fun androidArchiveName(pythonVersion: String, arch: String): String = "python-${pythonVersion}t-$arch-linux-android.tar.gz" fun url(pythonVersion: String, assetName: String): String = diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt index c0dc4a075..7daae5b07 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/AcquiredHeaderVersionTest.kt @@ -36,7 +36,7 @@ class AcquiredHeaderVersionTest { try { for (t in CPythonIncludeLayout.allTargets) { val v = if (t.startsWith("ios")) "3.14.6" else "3.14.7" - root.resolve(CPythonIncludeLayout.relativePath(t, CPythonFlavour.GIL, "3.14.7")) + root.resolve(CPythonIncludeLayout.relativePath(t, CPythonIncludeLayout.embeddedFlavour(t), "3.14.7")) .also { it.mkdirs() }.resolve("patchlevel.h").writeText(patchlevel(v)) } val failures = AcquiredHeaderVersion.mismatches(root, "3.14.7") diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt index 156c2051c..d803ee893 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt @@ -22,13 +22,14 @@ class CPythonIncludeDirectoriesTest { assertEquals("linux-x86_64-freethreaded/python/include/python3.14t", l.relativePath("linux-x86_64", CPythonFlavour.FREE_THREADED, version)) assertEquals("windows-x86_64/python/include", l.relativePath("windows-x86_64", CPythonFlavour.GIL, version)) assertEquals("windows-x86_64-freethreaded/python/include", l.relativePath("windows-x86_64", CPythonFlavour.FREE_THREADED, version)) - // #313: the 3.15 iOS archive has headers only inside the framework. - assertEquals("ios/Python.xcframework/ios-arm64/Python.framework/Headers", l.relativePath("ios-arm64", CPythonFlavour.GIL, version)) + // #313: the iOS archive has headers only inside the framework. + assertEquals("ios/Python.xcframework/ios-arm64/Python.framework/Headers", l.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version)) assertEquals( "ios/Python.xcframework/ios-arm64_x86_64-simulator/Python.framework/Headers", - l.relativePath("ios-arm64_x86_64-simulator", CPythonFlavour.GIL, version), + l.relativePath("ios-arm64_x86_64-simulator", CPythonFlavour.FREE_THREADED, version), ) - assertEquals("android-aarch64/prefix/include/python3.14", l.relativePath("android-aarch64", CPythonFlavour.GIL, version)) + // #328: the 3.15t Android archive (`prefix/include/python3.15t`), unpacked beside a `-freethreaded` suffix. + assertEquals("android-aarch64-freethreaded/prefix/include/python3.14t", l.relativePath("android-aarch64", CPythonFlavour.FREE_THREADED, version)) } @Test @@ -41,12 +42,22 @@ class CPythonIncludeDirectoriesTest { } } + /** + * Issue #328: Android and iOS embed only the project's own 3.15t builds; there is no GIL mobile + * build to lay out. Red before: `supports` allowed only GIL off desktop, so the free-threaded + * assertions below failed, and the GIL ones passed the other way round. + */ @Test - fun freeThreadedDoesNotExistOffDesktop() { - assertFalse(CPythonIncludeLayout.supports("android-aarch64", CPythonFlavour.FREE_THREADED)) - assertFalse(CPythonIncludeLayout.supports("ios-arm64", CPythonFlavour.FREE_THREADED)) + fun mobileHasOnlyTheFreeThreadedBuild() { + for (t in CPythonIncludeLayout.androidTargets + CPythonIncludeLayout.iosTargets) { + assertTrue(CPythonIncludeLayout.supports(t, CPythonFlavour.FREE_THREADED), t) + assertFalse(CPythonIncludeLayout.supports(t, CPythonFlavour.GIL), t) + } assertFailsWith { - CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version) + CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.GIL, version) + } + assertFailsWith { + CPythonIncludeLayout.relativePath("android-x86_64", CPythonFlavour.GIL, version) } } @@ -62,8 +73,8 @@ class CPythonIncludeDirectoriesTest { val project = ProjectBuilder.builder().build() val root = project.layout.projectDirectory.dir("extracted/$version") val tasks = CPythonIncludeLayout.allTargets.associateWith { target -> - // Android and iOS are always GIL; desktop follows the configured flavour. - val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + // Android and iOS are always free-threaded (#328); desktop follows the configured flavour. + val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.FREE_THREADED project.tasks.register("fake_${target.replace('-', '_')}") { doLast { root.dir(CPythonIncludeLayout.relativePath(target, flavour, version)).asFile @@ -72,7 +83,7 @@ class CPythonIncludeDirectoriesTest { } } val dirs = CPythonIncludeDirectories(root, version) { target, flavour -> - val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.FREE_THREADED if (flavour == extracted) tasks[target] else null } return project to dirs @@ -105,30 +116,27 @@ class CPythonIncludeDirectoriesTest { assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.GIL)) val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.GIL) } assertTrue("#158" in e.message.orEmpty(), e.message) - assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.FREE_THREADED) } + val ios = assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.GIL) } + assertTrue("#328" in ios.message.orEmpty(), ios.message) } /** - * Issue #158: desktop embeds only the free-threaded build, so the default flavour of every - * accessor is the one the library embeds for that target. Red before: the default was - * `CPythonFlavour.GIL` for every target, so `includeDir("macos-aarch64")` asked for a tree the - * free-threaded build does not extract and threw (and `embeddedFlavour` did not exist, so this - * does not compile against the old source). + * Issues #158 and #328: every target embeds only the free-threaded build, so the default + * flavour of every accessor is free-threaded. Red before #328: `embeddedFlavour` was GIL for + * Android and iOS, so the second loop and the Android path below failed. */ @Test fun theDefaultFlavourIsTheOneTheLibraryEmbeds() { - for (t in CPythonIncludeLayout.desktopTargets) { + for (t in CPythonIncludeLayout.allTargets) { assertEquals(CPythonFlavour.FREE_THREADED, CPythonIncludeLayout.embeddedFlavour(t), t) } - for (t in CPythonIncludeLayout.androidTargets + CPythonIncludeLayout.iosTargets) { - assertEquals(CPythonFlavour.GIL, CPythonIncludeLayout.embeddedFlavour(t), t) - } val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) assertTrue(dirs.isAvailable("macos-aarch64")) val root = dirs.includeDir("macos-aarch64").get().asFile.parentFile.parentFile.parentFile.parentFile assertEquals(File(root, "macos-aarch64-freethreaded/python/include/python3.14t"), dirs.includeDir("macos-aarch64").get().asFile) - assertEquals(File(root, "android-aarch64/prefix/include/python3.14"), dirs.includeDir("android-aarch64").get().asFile) + assertEquals(File(root, "android-aarch64-freethreaded/prefix/include/python3.14t"), dirs.includeDir("android-aarch64").get().asFile) assertEquals("python314t.lib", dirs.libraryFile("windows-x86_64").get().asFile.name) + assertEquals("libpython3.14t.so", dirs.libraryFile("android-x86_64").get().asFile.name) } @Test @@ -143,11 +151,11 @@ class CPythonIncludeDirectoriesTest { @Test fun linkLibraryLayoutMatchesTheObservedTree() { val l = CPythonIncludeLayout - assertEquals("android-aarch64/prefix/lib", l.linkLibraryDirRelativePath("android-aarch64", CPythonFlavour.GIL)) - assertEquals("android-x86_64/prefix/lib", l.linkLibraryDirRelativePath("android-x86_64", CPythonFlavour.GIL)) + assertEquals("android-aarch64-freethreaded/prefix/lib", l.linkLibraryDirRelativePath("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertEquals("android-x86_64-freethreaded/prefix/lib", l.linkLibraryDirRelativePath("android-x86_64", CPythonFlavour.FREE_THREADED)) assertEquals("windows-x86_64/python/libs", l.linkLibraryDirRelativePath("windows-x86_64", CPythonFlavour.GIL)) assertEquals("windows-x86_64-freethreaded/python/libs", l.linkLibraryDirRelativePath("windows-x86_64", CPythonFlavour.FREE_THREADED)) - assertEquals("libpython3.14.so", l.linkLibraryFileName("android-aarch64", CPythonFlavour.GIL, version)) + assertEquals("libpython3.14t.so", l.linkLibraryFileName("android-aarch64", CPythonFlavour.FREE_THREADED, version)) assertEquals("python314.lib", l.linkLibraryFileName("windows-x86_64", CPythonFlavour.GIL, version)) assertEquals("python314t.lib", l.linkLibraryFileName("windows-x86_64", CPythonFlavour.FREE_THREADED, version)) } @@ -162,10 +170,13 @@ class CPythonIncludeDirectoriesTest { } @Test - fun linkLibraryFreeThreadedOnlyOnWindows() { + fun linkLibraryAndroidIsFreeThreadedOnly() { val l = CPythonIncludeLayout assertTrue(l.supportsLinkLibrary("windows-x86_64", CPythonFlavour.FREE_THREADED)) - assertFalse(l.supportsLinkLibrary("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertTrue(l.supportsLinkLibrary("windows-x86_64", CPythonFlavour.GIL)) + // #328: Android links `libpython3.15t.so`; there is no GIL Android build. + assertTrue(l.supportsLinkLibrary("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertFalse(l.supportsLinkLibrary("android-x86_64", CPythonFlavour.GIL)) assertFalse(l.supportsLinkLibrary("macos-aarch64", CPythonFlavour.GIL)) assertFailsWith { l.linkLibraryDirRelativePath("macos-aarch64", CPythonFlavour.GIL) } } @@ -201,7 +212,7 @@ class CPythonIncludeDirectoriesTest { assertFalse(dirs.isLinkAvailable("windows-x86_64", CPythonFlavour.GIL)) val e = assertFailsWith { dirs.libraryDir("windows-x86_64", CPythonFlavour.GIL) } assertTrue("#158" in e.message.orEmpty(), e.message) - assertFailsWith { dirs.libraryDir("android-aarch64", CPythonFlavour.FREE_THREADED) } + assertFailsWith { dirs.libraryDir("android-aarch64", CPythonFlavour.GIL) } } @Test diff --git a/python-multiplatform-ksp/fixtures/app/build.gradle.kts b/python-multiplatform-ksp/fixtures/app/build.gradle.kts index 53bd8351a..d37be1eb5 100644 --- a/python-multiplatform-ksp/fixtures/app/build.gradle.kts +++ b/python-multiplatform-ksp/fixtures/app/build.gradle.kts @@ -23,10 +23,11 @@ kotlin { val downloadDir = project(":python-multiplatform").layout.buildDirectory.dir("python-standalone").get().asFile val pyVersion = project.findProperty("pythonVersion")?.toString() ?: project.rootProject.version.toString() val libVersion = pyVersion.split('.').subList(0, 2).joinToString(".") - // Version-keyed, matching python-multiplatform's `extractedDir`. - val targetExtractDir = "$downloadDir/extracted/$pyVersion/android-aarch64/prefix" + // Version-keyed, matching python-multiplatform's `extractedDir`; the free-threaded 3.15t + // tree (`android-aarch64-freethreaded`, `libpython3.15t.so`), the only Android build (#328). + val targetExtractDir = "$downloadDir/extracted/$pyVersion/android-aarch64-freethreaded/prefix" binaries.getTest(org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG).linkerOpts.addAll( - listOf("-L$targetExtractDir/lib/", "-lpython$libVersion", "-Wl,--allow-shlib-undefined"), + listOf("-L$targetExtractDir/lib/", "-lpython${libVersion}t", "-Wl,--allow-shlib-undefined"), ) } // The *second* Native leaf, and it is here for one reason: two of them make the default diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index cdce9f14b..a51654833 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -65,9 +65,11 @@ val configuredPythonVersion = project.property("pythonVersion").toString() // Issue #158, maintainer 2026-10-04: "Free-threaded is the default and the only build. The GIL // build is not kept as an option. Baseline is CPython 3.15t." Desktop (JVM, macosArm64, linuxX64) // therefore always acquires python-build-standalone's free-threaded build, and the -// `-PpythonFreeThreaded` switch that used to choose it is gone. Android and iOS still embed -// python.org's GIL build until #328, wasm until #175; which build each platform embeds is the -// `embedsFreeThreadedPython` expect/actual in the library, not a build-wide flag. +// `-PpythonFreeThreaded` switch that used to choose it is gone. Android (arm64-v8a, x86_64), +// androidNative and iOS embed the project's own 3.15t builds from the GitHub release +// `cpython-t-mobile` (#328; leader 2026-11-13: no opt-in flag on mobile, 3.15t is the +// only build). Only wasm still embeds a GIL build, until #175. Which build each platform embeds is +// the `embedsFreeThreadedPython` expect/actual in the library, not a build-wide flag. if (project.findProperty("pythonFreeThreaded") != null) { throw GradleException( "-PpythonFreeThreaded was removed (issue #158): desktop always embeds the free-threaded " + @@ -75,7 +77,6 @@ if (project.findProperty("pythonFreeThreaded") != null) { ) } val pbsRelease = project.findProperty("pythonBuildStandaloneRelease")?.toString() ?: "20261013" -val pythonAppleSupportBuild = project.findProperty("pythonAppleSupportBuild")?.toString() ?: "b11" val pythonVersion = configuredPythonVersion val libraryVersion = "$pythonVersion-${project.property("libraryVersionSuffix")}" // the root's publishedVersion (#70) @@ -91,7 +92,7 @@ buildkonfig { // No free-threaded field: which build a platform embeds differs by platform (#158), so it // is the `embedsFreeThreadedPython` expect/actual (commonMain `EmbeddedPythonBuild.kt`), // which `Versions.abiFlags` reads. One BuildConfig field shared by every target could not - // say "desktop 3.15t, Android 3.15". + // say "every native platform 3.15t, wasm GIL". } } @@ -100,19 +101,60 @@ val libVersion = pythonVersion.split('.').subList(0, 2).joinToString(".") val desktopAbiSuffix = "t" /** - * The `ftp/python//` directory holding a release, which drops any pre-release suffix: - * `3.15.0rc1` is published under `3.15.0`. Identical to the version itself for a final release, - * so it is always the right thing to build a python.org URL from. + * ABI flags of the Android, androidNative and iOS CPython: `t`. They embed the project's own + * free-threaded builds (#163, #328); there is no GIL mobile build (#158). + */ +val mobileAbiSuffix = "t" + +/** `3.15t`: the name of the mobile `libpython3.15t.so`, `lib/python3.15t/` and `include/python3.15t/`. */ +val mobileTag = "$libVersion$mobileAbiSuffix" + +/** + * Where the project's own free-threaded Android and iOS archives are published (#163, + * `build-ft-mobile.yml`): the GitHub release `cpython-t-mobile`, because neither + * python.org nor BeeWare publishes a free-threaded mobile build. The plugin's iOS consumer wiring + * (`FreeThreadedMobileRelease` in `IosConsumerWiring.kt`) reads the same release. + */ +val freeThreadedMobileReleaseUrl = + "https://github.com/thisisthepy/python-multiplatform/releases/download/cpython-${configuredPythonVersion}t-mobile" + +/** + * Suffix of the mobile lock keys (`android-aarch64-3.15.0-freethreaded`, `ios-3.15.0-freethreaded`) + * and of the Android extraction directories. The suffix is in the directory name for the same + * reason as [desktopFlavourSuffix]: an `android-aarch64/` GIL tree unpacked by an older checkout + * must never be mistaken for the free-threaded one. + */ +val mobileFlavourSuffix = "-freethreaded" + +/** Lock-key suffix of the mobile archives in `python-checksums.properties`. */ +val mobileLockKeySuffix = "-freethreaded" + +/** + * Fails the build when an extracted mobile CPython is not a free-threaded build: every `pyconfig` + * header in it must define `Py_GIL_DISABLED 1`. The archive name and the lockfile say what was + * meant to be downloaded; this says what was unpacked. * - * Declared here, above every consumer, because both the Android and the iOS download tasks need - * it. It used to sit below the Android tasks, which therefore built their URLs from the raw - * version and 404'd on any pre-release -- `ftp/python/3.15.0rc1/` does not exist. That stayed - * invisible because the archives were already sitting in the download directory, so the fetch was - * skipped; adding the `.sigstore` fetch is what surfaced it. + * The iOS framework's `Headers/pyconfig.h` is a 107-byte wrapper that includes + * `pyconfig-arm64.h` / `pyconfig-x86_64.h` by architecture (observed in the 3.15t archive), so a + * header that only includes another `pyconfig-*` is skipped and the per-architecture ones checked. */ -val pythonOrgReleaseDir = pythonVersion.split(".").let { parts -> - val patch = parts.getOrNull(2)?.takeWhile { it.isDigit() } ?: "0" - "${parts[0]}.${parts[1]}.$patch" +fun requireFreeThreadedHeaders(label: String, extracted: File) { + val name = Regex("""pyconfig(-[A-Za-z0-9_]+)?\.h""") + val wrapper = Regex("""#\s*include\s+["<]pyconfig-""") + val configs = extracted.walkTopDown() + .filter { it.isFile && name.matches(it.name) } + .filterNot { wrapper.containsMatchIn(it.readText()) } + .toList() + if (configs.isEmpty()) throw GradleException("$label: no pyconfig header under $extracted, cannot verify that it is free-threaded") + val re = Regex("""^\s*#\s*define\s+Py_GIL_DISABLED\s+1\b""", RegexOption.MULTILINE) + for (h in configs) { + if (!re.containsMatchIn(h.readText())) { + throw GradleException( + "$label: $h does not define Py_GIL_DISABLED 1. Android and iOS embed only the free-threaded " + + "CPython ${mobileTag} (#158, #328).", + ) + } + } } println("----------------------------------------------------------------------------------------") @@ -205,13 +247,13 @@ fun verifyChecksum(key: String, archive: File) { } // ================================================================================================= -// Sigstore verification of the python.org archives (ROADMAP §12, "Download integrity"). +// Sigstore verification of the downloaded CPython archives (ROADMAP §12, "Download integrity"). // // This is a SECOND gate, not a replacement for `verifyChecksum` above. The two prove different // things and neither implies the other: // // the lockfile "these are the exact bytes this repository reviewed and pinned" -// Sigstore "these are the bytes the CPython release manager actually signed" +// Sigstore "these are the bytes the publisher's release workflow actually signed" // // The lockfile is also the only one of the two that works offline, and the only one that covers // every source. So it stays unconditional and Sigstore is layered on top of it. @@ -228,21 +270,18 @@ fun verifyChecksum(key: String, archive: File) { // // ### What is covered, and what is not // -// python.org Android aarch64/x86_64 COVERED -- sibling `.sigstore` bundle -// python.org iOS XCframework (3.15+) COVERED -- same -// astral-sh/python-build-standalone NOT covered. It publishes no sibling signature at all: -// 853 release assets, and the only non-archive among them -// is `SHA256SUMS` (checked directly against the release). -// Its provenance lives in GitHub's attestations API, keyed -// by artifact *digest* rather than filename, and that API -// is rate-limited to 60 requests/hour unauthenticated. That -// is a different mechanism, not this one. The desktop path -// already verifies against the release's own SHA256SUMS in -// addition to the lockfile. -// beeware/Python-Apple-support NOT coverable. The release publishes five tar.gz assets -// and nothing else -- no checksums, no signatures -- and it -// has no attestations either. The SHA-256 pin is the only -// honest instrument available for iOS <= 3.14. +// astral-sh/python-build-standalone COVERED through GitHub's attestations API, keyed by the +// (desktop) artifact digest (`maybeVerifyPbsSigstore` below). The +// release also publishes `SHA256SUMS`, which the desktop +// path checks unconditionally in addition to the lockfile. +// thisisthepy cpython-t-mobile NOT covered. The project's own free-threaded Android and +// (Android, iOS; #163, #328) iOS archives carry no `.sigstore` bundle; the release has a +// `SHA256SUMS`, and the lockfile pins the same values (checked +// when the pins were written). The SHA-256 pin is the gate. +// +// python.org's Android and iOS archives used to be covered here through their sibling +// `.sigstore` bundles. They were GIL builds, and the library no longer acquires them +// (#158: "The GIL build is not kept as an option"; #328), so that path is gone. val verifySigstore = project.findProperty("verifyPythonSignatures")?.toString()?.toBoolean() ?: false val sigstoreVerifierClasspath: Configuration by configurations.creating { @@ -255,33 +294,9 @@ dependencies { add("sigstoreVerifierClasspath", "dev.sigstore:sigstore-java:2.2.0") } -/** The Fulcio certificate identity a python.org bundle must carry. */ +/** The Fulcio certificate identity a Sigstore bundle must carry. */ data class SigstoreIdentity(val subjectAlternativeName: String, val issuer: String) -/** - * Which release manager signed a given CPython version, per https://www.python.org/download/sigstore/ - * - * This is a map rather than a constant on purpose. The signing identity is a property of the - * *release manager*, and it changes between release series: 3.13 was Thomas Wouters signing via - * Google, 3.14 and 3.15 are Hugo van Kemenade signing via GitHub. A hardcoded identity would keep - * verifying happily on the version it was written for and then either fail confusingly or -- far - * worse -- be loosened by the next person to "just make it pass" on an older one. - * - * An unknown series is a hard failure rather than a skip. A verification step that silently - * declines to verify is the thing this whole section exists to avoid. - */ -fun pythonOrgSigstoreIdentity(version: String): SigstoreIdentity { - val series = version.split(".").take(2).joinToString(".") - return when (series) { - "3.14", "3.15" -> SigstoreIdentity("hugo@python.org", "https://github.com/login/oauth") - "3.12", "3.13" -> SigstoreIdentity("thomas@python.org", "https://accounts.google.com") - else -> throw GradleException( - "No Sigstore signing identity is recorded for CPython $series. Add it from " + - "https://www.python.org/download/sigstore/ rather than disabling verification." - ) - } -} - /** * Verifies `archive` against its sibling `.sigstore` bundle, pinning the signer identity. * @@ -370,17 +385,6 @@ fun verifySigstoreBundle(archive: File, bundleUrl: String?, bundleFile: File, id } } -/** Verifies a python.org archive if `-PverifyPythonSignatures=true`, otherwise says it skipped. */ -fun maybeVerifySigstore(archive: File, archiveUrl: String) { - if (!verifySigstore) return - verifySigstoreBundle( - archive = archive, - bundleUrl = "$archiveUrl.sigstore", - bundleFile = file("$downloadDir/${archive.name}.sigstore"), - identity = pythonOrgSigstoreIdentity(configuredPythonVersion) - ) -} - /** Verifies a python-build-standalone archive using GitHub Attestations API if `-PverifyPythonSignatures=true`. */ fun maybeVerifyPbsSigstore(archive: File, actualHash: String) { if (!verifySigstore) return @@ -547,18 +551,27 @@ val androidTargets = mapOf( "android-x86_64" to "x86_64" ) +/** + * Android embeds the project's own free-threaded CPython 3.15t for both ABIs it supports, + * arm64-v8a and x86_64 (#328; the owner approved the x86_64 3.15t build on 2026-11-13, #396, so + * both ABIs stay supported). The archives are the release `cpython-t-mobile`'s + * `python-t--linux-android.tar.gz`: a `prefix/` with `libpython3.15t.so`, + * `include/python3.15t` and `lib/python3.15t`, the same layout python.org's GIL archives had with + * the `t` added. There is no GIL Android build (#158). + */ val androidDownloadTasks = androidTargets.map { (platform, arch) -> - val url = "https://www.python.org/ftp/python/$pythonOrgReleaseDir/python-$configuredPythonVersion-$arch-linux-android.tar.gz" - val archive = file("$downloadDir/python-$configuredPythonVersion-$arch-linux-android.tar.gz") - val extractDir = file("$extractedDir/$platform") - val lockKey = "$platform-$configuredPythonVersion" + val archiveName = "python-${configuredPythonVersion}t-$arch-linux-android.tar.gz" + val url = "$freeThreadedMobileReleaseUrl/$archiveName" + val archive = file("$downloadDir/$archiveName") + val extractDir = file("$extractedDir/$platform$mobileFlavourSuffix") + val lockKey = "$platform-$configuredPythonVersion$mobileLockKeySuffix" pythonArchiveKeys[lockKey] = archive - + val taskName = "downloadPython_${platform.replace("-", "_")}" tasks.register(taskName) { inputs.property("url", url) outputs.dir(extractDir) - + doLast { if (!archive.exists()) { println("Downloading $url") @@ -569,16 +582,11 @@ val androidDownloadTasks = androidTargets.map { (platform, arch) -> } } } - - // python.org publishes no SHA256SUMS, so the lockfile is what pins these bytes. It also - // publishes a sibling `.sigstore` bundle, which `maybeVerifySigstore` checks - // against the release manager's pinned identity when `-PverifyPythonSignatures=true`. - // (An earlier comment here claimed Sigstore verification was "unreasonable in pure - // Gradle". It is not -- `dev.sigstore:sigstore-java` does it in-process; see the - // section above `updatePythonChecksums` for why it is opt-in rather than always on.) + + // The lockfile is the gate (see "What is covered" above): the release carries no + // Sigstore bundle, and its SHA256SUMS agrees with the pinned values. verifyChecksum(lockKey, archive) - maybeVerifySigstore(archive, url) - + val isEmpty = extractDir.list()?.isEmpty() ?: true if (isEmpty) { println("Extracting $archive to $extractDir") @@ -588,51 +596,34 @@ val androidDownloadTasks = androidTargets.map { (platform, arch) -> } } requireHeaderVersion(platform, extractDir) + requireFreeThreadedHeaders(platform, extractDir) } } } /** - * Which project publishes the iOS `Python.xcframework` for the configured version. - * - * python.org began publishing an official iOS XCframework with 3.15 (the first entries in - * `ftp/python/3.15.0/` are the 3.15.0b1 betas). BeeWare's Python-Apple-support, which was the - * only source before that, stops at `3.14-b11` (3.14.7) and has no 3.15 tag. The two do not overlap: - * 3.14 and earlier can only come from BeeWare, 3.15 and later only from python.org. So this is a - * hard switch on the version, not a preference. + * iOS embeds the project's own free-threaded `Python.xcframework` (#163, #328): the release + * `cpython-t-mobile`'s `python-t-iOS-XCframework.tar.gz`, the archive the + * plugin's iOS consumer wiring also downloads (SPEC T-2a), so a consumer never pairs this klib with + * a different framework. Slices `ios-arm64` and `ios-arm64_x86_64-simulator`, each with + * `Python.framework/Headers` (a `pyconfig.h` that includes `pyconfig-.h`, which defines + * `Py_GIL_DISABLED 1`) and `lib-/python3.15t/`; the pure-Python stdlib is + * `Python.xcframework/lib/python3.15t/`. * - * Swapping is otherwise cheap, because the trees are layout-compatible everywhere this build - * reaches into them -- `Python.xcframework//Python.framework/Headers`, `-F .../`, - * `Python.xcframework/lib/pythonX.Y` and `...//lib-arm64/pythonX.Y` all exist in both. - * The differences are in parts nothing here reads: BeeWare adds a `platform-config/` directory - * (cross-compilation sysconfig data for building wheels) and a `VERSIONS` file, and its headers - * still carry `module.modulemap`, `lock.h`, `monitoring.h` and `typeslots.h` where 3.15 has - * `pyabi.h`, `slots.h` and `slots_generated.h` instead -- a 3.14-vs-3.15 difference, not a - * packaging one. Both ship a GIL-enabled build; neither publishes a free-threaded iOS variant. + * There is no GIL iOS build and no switch (#158; leader 2026-11-13). The python.org (3.15) and + * BeeWare (3.14 and earlier) GIL sources this build used to choose between are gone. */ -val iosFromPythonOrg = pythonVersion.split(".").let { - it[0].toInt() > 3 || (it[0].toInt() == 3 && it[1].toInt() >= 15) -} - -val iosArchiveName = if (iosFromPythonOrg) { - "python-$pythonVersion-iOS-XCframework.tar.gz" -} else { - "Python-$libVersion-iOS-support.$pythonAppleSupportBuild.tar.gz" -} -val iosUrl = if (iosFromPythonOrg) { - "https://www.python.org/ftp/python/$pythonOrgReleaseDir/$iosArchiveName" -} else { - "https://github.com/beeware/Python-Apple-support/releases/download/$libVersion-$pythonAppleSupportBuild/$iosArchiveName" -} +val iosArchiveName = "python-${configuredPythonVersion}t-iOS-XCframework.tar.gz" +val iosUrl = "$freeThreadedMobileReleaseUrl/$iosArchiveName" val iosArchive = file("$downloadDir/$iosArchiveName") val iosExtractDir = file("$extractedDir/ios") -val iosLockKey = if (iosFromPythonOrg) "ios-$pythonVersion-pythonorg" else "ios-$libVersion-$pythonAppleSupportBuild" +val iosLockKey = "ios-$configuredPythonVersion$mobileLockKeySuffix" pythonArchiveKeys[iosLockKey] = iosArchive val downloadPython_ios = tasks.register("downloadPython_ios") { inputs.property("url", iosUrl) outputs.dir(iosExtractDir) - + doLast { if (!iosArchive.exists()) { println("Downloading $iosUrl") @@ -643,24 +634,14 @@ val downloadPython_ios = tasks.register("downloadPython_ios") { } } } - - // The two iOS sources are not equally verifiable, and the difference is worth stating. - // - // BeeWare (<= 3.14) publishes five tar.gz assets and nothing else -- no checksums, no - // signatures, no GitHub attestations. The lockfile pin is the only honest instrument - // there, and no amount of build wiring changes that. - // - // python.org (3.15+) publishes a sibling `.sigstore` bundle, which IS verifiable - // here. An earlier comment claimed otherwise; it was wrong. - // - // The lockfile applies to both regardless, because it is the only check that works - // offline and the only one that covers every source. + + // The lockfile is the gate (see "What is covered" above). verifyChecksum(iosLockKey, iosArchive) - if (iosFromPythonOrg) maybeVerifySigstore(iosArchive, iosUrl) - // The iOS extract dir is keyed by pythonVersion, not by archive, so bumping the pinned - // archive (b10 -> b11) would otherwise keep the old tree: "not empty, skip". A stamp of the - // archive's lock key + digest tells a stale tree from a current one (issue #47). + // The iOS extract dir is keyed by pythonVersion, not by archive, so changing the pinned + // archive (python.org's GIL framework -> the 3.15t one) would otherwise keep the old tree: + // "not empty, skip". A stamp of the archive's lock key and name tells a stale tree from a + // current one (issue #47). val iosStamp = file("$iosExtractDir/.archive-stamp") val iosWanted = "$iosLockKey ${iosArchive.name}" if (iosExtractDir.exists() && (!iosStamp.exists() || iosStamp.readText().trim() != iosWanted)) { @@ -677,6 +658,7 @@ val downloadPython_ios = tasks.register("downloadPython_ios") { iosStamp.writeText(iosWanted + "\n") } requireHeaderVersion("ios", iosExtractDir) + requireFreeThreadedHeaders("ios", iosExtractDir) } } @@ -692,7 +674,7 @@ val downloadAllPythonBuilds by tasks.registering { * TypedPython's C build) read it with * `project(":python-multiplatform").extensions.getByType().includeDir("macos-aarch64")` * after `evaluationDependsOn(":python-multiplatform")`. Only the flavour the library embeds has a - * task: free-threaded on desktop (#158), GIL on Android and iOS (#328). See + * task, and that is free-threaded on every target: desktop (#158), Android and iOS (#328). See * `docs/platforms/python-version-acquisition.md` §7. */ extensions.add( @@ -711,18 +693,19 @@ extensions.add( * (`META-INF/python-multiplatform/python.properties`) in the jar and the AAR. See * `docs/platforms/python-version-acquisition.md` "Published version". * - * Per platform since #158: the desktop jar (and the macosArm64/linuxX64 klibs) embed 3.15t, the AAR, - * the iOS and androidNative klibs the GIL 3.15 until #328, wasm GIL until #175. So the - * free-threaded attribute is decided per published variant, and the jar and the AAR each get - * their own resource. Shared metadata variants carry no CPython and get no free-threaded attribute. - * The `pythonMultiplatform` extension holds one value; it is the desktop one (what composite-build - * consumers that compile against the desktop CPython read), and the per-variant attributes are the + * Per platform since #158: the desktop jar, the AAR and every Kotlin/Native klib (macosArm64, + * linuxX64, iOS, androidNative) embed 3.15t (#158, #328); only wasm still embeds a GIL build, until + * #175. So the free-threaded attribute is decided per published variant (false on the wasm ones), + * and the jar and the AAR each get their own resource. Shared metadata variants carry no CPython and + * get no free-threaded attribute. The `pythonMultiplatform` extension holds one value, the + * free-threaded one every non-wasm platform embeds, and the per-variant attributes are the * per-platform truth. (Extension and metadata rules: agent proposal, not confirmed.) */ -val desktopEmbeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = true) -val androidEmbeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = false) -val freeThreadedVariantPrefixes = listOf("desktop", "macosArm64", "linuxX64") -desktopEmbeddedPythonVersion.register(project) { configurationName -> +val embeddedPythonVersion = EmbeddedPythonVersion(configuredPythonVersion, freeThreaded = true) +// `android` covers the AAR's variants and androidNativeArm64/X64, `ios` iosArm64, iosX64 and +// iosSimulatorArm64. Anything else that is not metadata (the wasmJs variants) is GIL. +val freeThreadedVariantPrefixes = listOf("desktop", "macosArm64", "linuxX64", "android", "ios") +embeddedPythonVersion.register(project) { configurationName -> when { freeThreadedVariantPrefixes.any { configurationName.startsWith(it) } -> true configurationName.startsWith("metadata") || "Metadata" in configurationName -> null @@ -743,10 +726,10 @@ fun registerPythonProperties(name: String, embedded: EmbeddedPythonVersion) = ta } /** `freeThreaded=true`: the desktop jar's resource. */ -val generateDesktopPythonProperties = registerPythonProperties("generateDesktopPythonProperties", desktopEmbeddedPythonVersion) +val generateDesktopPythonProperties = registerPythonProperties("generateDesktopPythonProperties", embeddedPythonVersion) -/** `freeThreaded=false`: the AAR's resource (GIL until #328). */ -val generateAndroidPythonProperties = registerPythonProperties("generateAndroidPythonProperties", androidEmbeddedPythonVersion) +/** `freeThreaded=true`: the AAR's resource (3.15t since #328). */ +val generateAndroidPythonProperties = registerPythonProperties("generateAndroidPythonProperties", embeddedPythonVersion) /** * The dev agent (SPEC L-12, #233) is Python, kept as a real file -- `src/commonMain/python/ @@ -1108,7 +1091,7 @@ kotlin { into("$androidBuildDir/jniLibs/") abiList.forEach { abi -> val arch = if (abi == "arm64-v8a") "aarch64" else "x86_64" - from("$extractedDir/android-$arch/prefix/lib") { + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib") { include("libpython*.so") include("lib*_python.so") into(abi) @@ -1129,7 +1112,7 @@ kotlin { // The library itself is 0.49 MB of that. Two of the entries above are payload nothing // on a device ever reads, and both are dropped here: // - // - `include/python$libVersion` is CPython's C headers. cinterop reads them from the + // - `include/python$mobileTag` is CPython's C headers. cinterop reads them from the // extraction tree (`targetIncludePath`), never from assets, and no Kotlin source in // `androidMain`, `artMain`, `androidInstrumentedTest` or `sample` opens an asset // under `include/`. They were being shipped to every device for nothing. @@ -1154,10 +1137,12 @@ kotlin { doFirst { delete("$androidBuildDir/assets/") } abiList.forEach { abi -> val arch = if (abi == "arm64-v8a") "aarch64" else "x86_64" - from("$extractedDir/android-$arch/prefix/lib/python$libVersion") { - exclude("config-$libVersion-$arch-linux-android/") + // `lib/python3.15t`: the free-threaded stdlib, which is where + // `PythonBootstrap.stdlibRelativePath()` (taggedVersionString) looks (#328). + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib/python$mobileTag") { + exclude("config-$mobileTag-$arch-linux-android/") exclude("test/") - into("$abi/lib/python$libVersion") + into("$abi/lib/python$mobileTag") } } } @@ -1284,13 +1269,13 @@ kotlin { else -> throw RuntimeException("Unsupported ABI: $konanTarget") } val targetExtractDir = when(konanTarget.family) { - Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}/prefix" + Family.ANDROID -> "$extractedDir/android-${if (targetABI == "arm64-v8a") "aarch64" else "x86_64"}$mobileFlavourSuffix/prefix" Family.IOS -> "$extractedDir/ios/Python.xcframework" Family.OSX, Family.LINUX -> "$extractedDir/$targetABI$desktopFlavourSuffix/python" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") } val targetIncludePath = when(konanTarget.family) { - Family.ANDROID -> "$targetExtractDir/include/python$libVersion" + Family.ANDROID -> "$targetExtractDir/include/python$mobileTag" Family.IOS -> "$targetExtractDir/$targetABI/Python.framework/Headers" Family.OSX, Family.LINUX -> "$targetExtractDir/include/python$libVersion$desktopAbiSuffix" else -> throw RuntimeException("Unsupported target family: ${konanTarget.family}") @@ -1313,8 +1298,11 @@ kotlin { binaries { if (konanTarget.family == Family.ANDROID) { - sharedLib("multiplatform_python$libVersion") { - linkerOpts.addAll(listOf("-L$targetExtractDir/lib/", "-lpython$libVersion", "-u", "JNI_OnLoad")) + // `libmultiplatform_python3.15t.so`: the name `manager.loadLibPython` asks for on + // Android ("multiplatform_python" + taggedVersionString), linked against the + // free-threaded `libpython3.15t.so` (#328). + sharedLib("multiplatform_python$mobileTag") { + linkerOpts.addAll(listOf("-L$targetExtractDir/lib/", "-lpython$mobileTag", "-u", "JNI_OnLoad")) linkTaskProvider.configure { val type = if (buildType == NativeBuildType.DEBUG) "debug" else "release" @@ -1344,8 +1332,8 @@ kotlin { } } getTest(NativeBuildType.DEBUG).linkerOpts.addAll(listOf( - "-L$targetExtractDir/lib/", - "-lpython$libVersion", + "-L$targetExtractDir/lib/", + "-lpython$mobileTag", "-Wl,--allow-shlib-undefined", "-u", "JNI_OnLoad" )) @@ -1542,18 +1530,18 @@ tasks.named("stageAndroidNative${targetSuffix}TestPayload") { dependsOn(downloadAllPythonBuilds) - from("$extractedDir/android-$arch/prefix/lib") { + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib") { include("*.so") include("*.so.*") into("lib") } - from("$extractedDir/android-$arch/prefix/lib/python$libVersion") { - exclude("config-$libVersion-*/") - into("lib/python$libVersion") + from("$extractedDir/android-$arch$mobileFlavourSuffix/prefix/lib/python$mobileTag") { + exclude("config-$mobileTag-*/") + into("lib/python$mobileTag") } into(stagingDir) includeEmptyDirs = false diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt new file mode 100644 index 000000000..6fffe2e85 --- /dev/null +++ b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidFreeThreadedBuildTest.kt @@ -0,0 +1,71 @@ +package python.multiplatform + +import python.multiplatform.env.PythonBootstrap +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.PythonTestFixture +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * SPEC T-2 (issue #328): Android (ART, arm64-v8a and x86_64) embeds the free-threaded CPython 3.15t, + * and only that build. + * + * The maintainer's decision on #158 (2026-10-04): "Free-threaded is the default and the only build. + * The GIL build is not kept as an option. Baseline is CPython 3.15t." Desktop moved first (#398); + * Android follows here, with no opt-in flag (leader, 2026-11-13). + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `androidMain`, so `abiFlags` was + * `""`, the stdlib was looked for under `lib/python3.15`, the AAR's `python.properties` said + * `freeThreaded=false`, `autoDrainInterval` defaulted to 0, and the loaded `libpython3.15.so` was + * python.org's GIL build (`Py_GIL_DISABLED` 0). Each assertion fails on that build with its own + * message. (This replaces `AndroidGilBuildTest`, which pinned the GIL build until #328.) + */ +class AndroidFreeThreadedBuildTest { + + @Test + fun androidReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "Android embeds CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theStdlibIsLookedForUnderTheFreeThreadedDirectory() { + val tagged = Versions.currentVersion.compactVersionString + "t" + assertEquals("lib/python$tagged", PythonBootstrap.stdlibRelativePath()) + assertEquals(".python-multiplatform-stdlib.$tagged", PythonBootstrap.stampFile(File("/prefix")).name) + } + + @Test + fun theAarSaysItIsFreeThreaded() { + // The AAR's `META-INF/python-multiplatform/python.properties` (generateAndroidPythonProperties). + val text = assertNotNull( + PythonBootstrap::class.java.getResourceAsStream("/META-INF/python-multiplatform/python.properties"), + "the AAR carries no META-INF/python-multiplatform/python.properties", + ).use { it.readBytes().decodeToString() } + assertTrue("freeThreaded=true" in text, "the AAR's python.properties must say freeThreaded=true (#328): $text") + } + + @Test + fun theLoadedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + // `Py_GIL_DISABLED` is a build-time fact of the libpython that was loaded; + // `sys._is_gil_enabled()` is not asserted, because an extension module without `Py_mod_gil` + // re-enables the GIL at run time on a free-threaded build. + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the Android libpython is a GIL build; Android must embed the free-threaded one (#328)", + ) + } + + @Test + fun theAutomaticCheckpointIsOnByDefault() { + assertTrue( + Python3.autoDrainInterval > 0, + "autoDrainInterval=${Python3.autoDrainInterval}: the free-threaded default turns the checkpoint on", + ) + } +} diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt deleted file mode 100644 index b9a3cc45c..000000000 --- a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/AndroidGilBuildTest.kt +++ /dev/null @@ -1,34 +0,0 @@ -package python.multiplatform - -import python.multiplatform.env.PythonBootstrap -import java.io.File -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Issue #158 moved desktop to the free-threaded CPython 3.15t; Android (ART) still embeds - * python.org's GIL CPython 3.15 until #328. This pins that the desktop change did not leak here. - * - * Not red before the change: Android was GIL before and is GIL after. It is the guard against the - * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which - * [PythonBootstrap.stdlibRelativePath] becomes `lib/python3.15t`, a directory the staged GIL - * prefix does not contain, and the app dies in `onCreate`. Nothing here loads the interpreter. It - * lives in `androidInstrumentedTest` because `androidUnitTest` does not compile on develop (four - * `commonTest` expects have no host-JVM actual), not because it needs a device. - */ -class AndroidGilBuildTest { - - @Test - fun androidStillReportsTheGilAbi() { - val v = Versions.currentVersion - assertEquals("", v.abiFlags, "Android embeds the GIL CPython until #328") - assertEquals(v.compactVersionString, v.taggedVersionString) - } - - @Test - fun theStdlibIsLookedForUnderTheGilDirectory() { - val compact = Versions.currentVersion.compactVersionString - assertEquals("lib/python$compact", PythonBootstrap.stdlibRelativePath()) - assertEquals(".python-multiplatform-stdlib.$compact", PythonBootstrap.stampFile(File("/prefix")).name) - } -} diff --git a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 02b17a19c..eb52a02d8 100644 --- a/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/androidInstrumentedTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -359,9 +359,13 @@ class CycleCollectionTest { * releases it. `pmp_proxy_dealloc` in `jni_onload.def` hand-writes that (`PyObject_Type` for * the temporary, then two `Py_DecRef`s), so it can leak one type reference per instance by * forgetting, or free the type out from under the process by doing it twice. Both are checked - * by reading `ob_refcnt` through [bindings.obRefCnt] -- the first field of `PyObject`, fixed - * there by the stable ABI. `sys.getrefcount` cannot answer this: it counts its own argument's - * temporary reference. + * by reading the type's reference count through [bindings.obRefCnt], which sums the + * free-threaded header's `ob_ref_local` and `ob_ref_shared` (Android embeds 3.15t, #328). + * `sys.getrefcount` cannot answer this: it counts its own argument's temporary reference. + * + * A heap type's references are deferred on the free-threaded build until a collection + * materialises them, so one is taken on either side of the instantiation loop, as in the + * desktop `CycleCollectionTest`. * * The "rises while alive" assertion is not decoration: it is what proves the probe reads a * real refcount, so the "returns afterwards" assertion cannot pass vacuously on an address @@ -373,6 +377,9 @@ class CycleCollectionTest { val proxyTypeAddr = ProxyTypeFactory.createProxyType() assertTrue(proxyTypeAddr != 0L, "Proxy type creation failed") + // Materialise the deferred references the type already carries (free-threaded build). + python.native.ffi.PyGC_Collect() + val liveBefore = HandleTable.liveCount val typeRefBefore = bindings.obRefCnt(proxyTypeAddr) @@ -395,6 +402,8 @@ class CycleCollectionTest { liveBefore + rounds, HandleTable.liveCount, "setup failed: $rounds handles should be rooted while the proxies are alive", ) + // The instantiations above were deferred; a collection makes them countable. + python.native.ffi.PyGC_Collect() val typeRefAlive = bindings.obRefCnt(proxyTypeAddr) assertEquals( typeRefBefore + rounds, typeRefAlive, diff --git a/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt index dd8d5e0fa..7d78bf149 100644 --- a/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt +++ b/python-multiplatform/src/androidMain/kotlin/python/multiplatform/EmbeddedPythonBuild.android.kt @@ -1,4 +1,4 @@ package python.multiplatform -/** Android (ART) embeds python.org's GIL CPython until #328. See the `expect` in commonMain. */ -internal actual val embedsFreeThreadedPython: Boolean = false +/** Android (ART) embeds the project's own free-threaded CPython 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt index 6c9f0f29c..585848d57 100644 --- a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt +++ b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/ShapeDowncalls.android.kt @@ -6,7 +6,7 @@ import java.nio.ByteBuffer * Android (ART) actuals for the shape vocabulary declared in `jvmMain/.../ShapeDowncalls.kt`. * * Each shape is one JNI native-method call into a `@CName`-exported trampoline compiled into - * `libmultiplatform_python3.13.so` (see `nativeMain/.../EmbedAPI.native.kt`), which casts the + * `libmultiplatform_python3.15t.so` (see `nativeMain/.../EmbedAPI.native.kt`), which casts the * `fn` address to a typed `CFunction` pointer and invokes it directly. This is "path 1" from * `docs/design/downcall-design.md`: ordinary JNI -> trampoline -> indirect call. ART cannot synthesize * an arbitrary native call at runtime the way Kotlin/Native's cinterop can at compile time, so diff --git a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt index 1a55088e5..6bd0160bc 100644 --- a/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt +++ b/python-multiplatform/src/androidMain/kotlin/python/native/ffi/bindings.kt @@ -686,12 +686,14 @@ object bindings { @JvmStatic external fun proxySetHandle(proxy: Long, handle: Long) /** - * `ob_refcnt` of the object at [obj], read straight out of the header. + * `Py_REFCNT` of the object at [obj], read straight out of the header. * * `Py_REFCNT` is a macro and `sys.getrefcount` adds its argument's own temporary to the * answer, so neither can say whether `tp_dealloc` released the instance's reference to its - * heap type exactly once. `ob_refcnt` is the first field of `PyObject` and its offset is - * fixed by the stable ABI, which is the one layout fact this file is allowed to know. + * heap type exactly once. Android embeds the free-threaded 3.15t (#328), whose `PyObject` + * header splits the count into `uint32 ob_ref_local` at +12 and `Py_ssize_t ob_ref_shared` at + * +16 (shifted left by 2); offset 0 is the owning thread id. The C side sums the two, as the + * desktop `CycleCollectionTest.typeRefCount` does. * * Composed on the C side for the same reason [proxySetHandle] is: reading 8 bytes at a raw * address from Kotlin/JVM would need `sun.misc.Unsafe`, a restricted non-SDK interface here. diff --git a/python-multiplatform/src/artMain/cinterop/jni_onload.def b/python-multiplatform/src/artMain/cinterop/jni_onload.def index 92f0e76e3..892b27061 100644 --- a/python-multiplatform/src/artMain/cinterop/jni_onload.def +++ b/python-multiplatform/src/artMain/cinterop/jni_onload.def @@ -1391,13 +1391,20 @@ static void proxy_set_handle(JNIEnv *e, jclass c, jlong obj, jlong handle) { } /* - * ob_refcnt of the object at `obj`, read straight out of the header. + * Py_REFCNT of the object at `obj`, read straight out of the header. * * Py_REFCNT is a macro over a struct this file does not include, and sys.getrefcount counts its * own argument's temporary reference -- neither can answer whether tp_dealloc released the * instance's reference to its heap type exactly once, which is what the cycle-collection tests - * check. ob_refcnt is the first field of PyObject and the stable ABI fixes that; it is 64-bit on - * both ABIs this project ships (arm64-v8a, x86_64), so jlong is the right width. + * check. + * + * androidNative links the free-threaded CPython 3.15t, and only that build (#158, #328). Its + * PyObject header (Include/object.h, Py_GIL_DISABLED) is not the GIL one: offset 0 is ob_tid, the + * owning thread's id, and the count is split between a thread-local uint32 ob_ref_local at +12 and + * an atomic Py_ssize_t ob_ref_shared at +16, shifted left by _Py_REF_SHARED_SHIFT (2). Py_REFCNT + * is their sum; this reproduces it, as the desktop CycleCollectionTest's typeRefCount does. Both + * ABIs this project ships (arm64-v8a, x86_64) are 64-bit, so the offsets hold for both. The shared + * half is read with a plain load, which is the same racy poll the tests already accept. * * A test probe. Composed here for the same reason proxy_set_handle is: reading 8 bytes at a raw * address from Kotlin/JVM needs sun.misc.Unsafe, a restricted non-SDK interface on Android. @@ -1407,7 +1414,10 @@ static void proxy_set_handle(JNIEnv *e, jclass c, jlong obj, jlong handle) { */ static jlong ob_refcnt(JNIEnv *e, jclass c, jlong obj) { if (obj == 0) return 0; - return *(jlong *)(intptr_t)obj; + const char *header = (const char *)(intptr_t)obj; + uint32_t local = *(const uint32_t *)(header + 12); + int64_t shared = *(const int64_t *)(header + 16); + return (jlong)local + (jlong)(shared >> 2); } /* diff --git a/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt index 79200f5e8..0123e05d8 100644 --- a/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt +++ b/python-multiplatform/src/artMain/kotlin/python/multiplatform/EmbeddedPythonBuild.art.kt @@ -1,4 +1,4 @@ package python.multiplatform -/** androidNative links python.org's GIL CPython until #328. See the `expect` in commonMain. */ -internal actual val embedsFreeThreadedPython: Boolean = false +/** androidNative links the project's own free-threaded CPython 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt new file mode 100644 index 000000000..6e3179511 --- /dev/null +++ b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeFreeThreadedBuildTest.kt @@ -0,0 +1,32 @@ +package python.multiplatform + +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-2 (issue #328): androidNative (`androidNativeArm64`, `androidNativeX64`) links the + * free-threaded CPython 3.15t, the same build as Android (ART). + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `artMain`, so `abiFlags` was `""`, + * and the cinterop compiled against `include/python3.15` and linked `-lpython3.15`, python.org's GIL + * build. (This replaces `AndroidNativeGilBuildTest`, which pinned the GIL build until #328.) + */ +class AndroidNativeFreeThreadedBuildTest { + + @Test + fun androidNativeReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "androidNative links CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLinkedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the androidNative libpython is a GIL build; it must be the free-threaded one (#328)", + ) + } +} diff --git a/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt b/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt deleted file mode 100644 index ca41ade40..000000000 --- a/python-multiplatform/src/artTest/kotlin/python/multiplatform/AndroidNativeGilBuildTest.kt +++ /dev/null @@ -1,22 +0,0 @@ -package python.multiplatform - -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Issue #158 moved desktop to the free-threaded CPython 3.15t; androidNative still embeds the GIL build until - * #328. This pins that the desktop change did not leak here. - * - * Not red before the change: androidNative was GIL before and is GIL after. It is the guard against the - * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which - * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that androidNative does not ship. - */ -class AndroidNativeGilBuildTest { - - @Test - fun androidNativeStillReportsTheGilAbi() { - val v = Versions.currentVersion - assertEquals("", v.abiFlags, "androidNative embeds the GIL CPython until #328") - assertEquals(v.compactVersionString, v.taggedVersionString) - } -} diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt index 9d2c69b44..b383d0f51 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/EmbeddedPythonBuild.kt @@ -7,22 +7,25 @@ package python.multiplatform * A property of the platform, not of the whole build. The maintainer's decision on issue #158 * (2026-10-04): "Free-threaded is the default and the only build. The GIL build is not kept as an * option. Baseline is CPython 3.15t." The desktop half landed first (leader, 2026-11-13: "flip the - * desktop default to 3.15t now, before the #377 alpha03 release"); Android and iOS still embed - * the GIL CPython 3.15 they download from python.org until #328, and wasm until #175. So: + * desktop default to 3.15t now, before the #377 alpha03 release"); Android, androidNative and iOS + * followed under #328 with the project's own 3.15t builds (release `cpython-t-mobile`; leader + * 2026-11-13: no opt-in flag on mobile). wasm still embeds a GIL build until #175. So: * * | Source set | Targets | Embeds | * |---------------------|---------------------------------|---------------| * | `desktopMain` | desktop JVM | 3.15t | * | `desktopNativeMain` | `macosArm64`, `linuxX64` | 3.15t | - * | `androidMain` | Android (ART) | 3.15 (GIL) | - * | `artMain` | `androidNativeArm64`, `X64` | 3.15 (GIL) | - * | `iosMain` | iOS | 3.15 (GIL) | + * | `androidMain` | Android (ART) | 3.15t | + * | `artMain` | `androidNativeArm64`, `X64` | 3.15t | + * | `iosMain` | iOS | 3.15t | * | `wasmJsMain` | `wasmJs` | GIL | * * It used to be one `BuildConfig.pythonFreeThreaded` shared by every target, which could not - * express this: flipping it for desktop made Android look for `lib/python3.15t`, a directory its - * GIL prefix does not have. A plain (non-`inline`) `expect val`, so the intermediate-source-set - * crash in `commonMain/README.md` does not apply. + * express the transition: flipping it for desktop made Android look for `lib/python3.15t`, a + * directory its GIL prefix did not have. With every non-wasm platform free-threaded it stays an + * `expect`, because wasm is still the exception (agent proposal, not confirmed: drop the + * `expect` once #175 makes wasm free-threaded too). A plain (non-`inline`) `expect val`, so the + * intermediate-source-set crash in `commonMain/README.md` does not apply. * * Everything CPython names after itself follows from this through [Versions.abiFlags]: the shared * library (`libpython3.15t`), the stdlib directory (`lib/python3.15t/`), and the PYTHONHOME check. diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt index b8cc66d2e..ef50c3d80 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt @@ -51,9 +51,9 @@ class Versions private constructor(val versionString: String) { * CPython's ABI flags suffix: `t` on a free-threaded build (PEP 703), empty otherwise. * * This is a property of the *build*, not of the version -- 3.15 ships in both flavours out of - * the same source tree -- and of the *platform*: desktop (JVM and Kotlin/Native) embeds 3.15t, - * while Android and iOS still embed the GIL build until #328 (issue #158). So it comes from the - * per-platform [embedsFreeThreadedPython] rather than from [versionString]. + * the same source tree -- and of the *platform*: desktop (JVM and Kotlin/Native), Android, + * androidNative and iOS embed 3.15t (issues #158, #328), wasm a GIL build until #175. So it + * comes from the per-platform [embedsFreeThreadedPython] rather than from [versionString]. */ val abiFlags: String get() = if (embedsFreeThreadedPython) "t" else "" diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt index d31fac6a9..b5ff2bbc5 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt @@ -810,8 +810,8 @@ def __pmp_run_file__(path, argv): * released since the previous one, so a workload that never drops a wrapper never pays. * * Which build that is depends on the platform ([python.multiplatform.embedsFreeThreadedPython]): - * desktop embeds 3.15t since issue #158, so this defaults to on there; Android and iOS embed the - * GIL build until #328, so it defaults to off there. + * desktop, Android, androidNative and iOS embed 3.15t (issues #158, #328), so this defaults to + * on there; wasm embeds a GIL build until #175, so it defaults to off there. * * Defaults to on for free-threaded builds, where deferred release is the defect described on * [drainPendingReleases], a build with the global lock frees a *plain* reference the diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt index 0cdc3aa8f..fc49790de 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/VersionsTest.kt @@ -14,11 +14,12 @@ import kotlin.test.assertTrue * a file that was downloaded and is sitting right there under a different name. * * These assertions are written against the per-platform [embedsFreeThreadedPython] rather than - * hard-coding one flavour, because the flavour differs by platform (issue #158): desktop embeds - * 3.15t, Android and iOS the GIL 3.15 until #328. The platform test source sets pin the literal - * value for their platform (`DesktopFreeThreadedBuildTest`, `DesktopNativeFreeThreadedBuildTest`, - * `AndroidGilBuildTest`, `AndroidNativeGilBuildTest`, `IosGilBuildTest`), which is what a wrong - * actual would fail. + * hard-coding one flavour, because the flavour differs by platform: desktop, Android, + * androidNative and iOS embed 3.15t (issues #158, #328), wasm a GIL build until #175. The platform + * test source sets pin the literal value for their platform (`DesktopFreeThreadedBuildTest`, + * `DesktopNativeFreeThreadedBuildTest`, `AndroidFreeThreadedBuildTest`, + * `AndroidNativeFreeThreadedBuildTest`, `IosFreeThreadedBuildTest`, `WasmGilBuildTest`), which is + * what a wrong actual would fail. */ class VersionsTest { diff --git a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt index 2e9c36f80..b96a99d3c 100644 --- a/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt +++ b/python-multiplatform/src/desktopNativeTest/kotlin/python/multiplatform/DesktopNativeFreeThreadedBuildTest.kt @@ -9,8 +9,8 @@ import kotlin.test.assertEquals * * Red before the change: `abiFlags` came from the one `BuildConfig.pythonFreeThreaded` (default * false), so it was `""` here, and the cinterop compiled against `include/python3.15` and linked - * `-lpython3.15`. This source set shares `nativeMain` with iOS and androidNative, which must stay - * GIL (`IosGilBuildTest`, `AndroidNativeGilBuildTest`), so a single shared flag cannot pass both. + * `-lpython3.15`. This source set shares `nativeMain` with iOS and androidNative, which embed 3.15t + * as well since #328 (`IosFreeThreadedBuildTest`, `AndroidNativeFreeThreadedBuildTest`). */ class DesktopNativeFreeThreadedBuildTest { diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt index 990f00e23..f8533abaf 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/DesktopFreeThreadedBuildTest.kt @@ -18,8 +18,9 @@ import kotlin.test.assertTrue * `Py_GIL_DISABLED = 0` and `autoDrainInterval` defaulted to 0. Each assertion below fails on that * build with its own message. * - * What this cannot find: whether Android or iOS were left alone. Their literal values are pinned in - * their own test source sets (`AndroidGilBuildTest`, `AndroidNativeGilBuildTest`, `IosGilBuildTest`). + * What this cannot find: what Android, androidNative, iOS and wasm embed. Their literal values are + * pinned in their own test source sets (`AndroidFreeThreadedBuildTest`, + * `AndroidNativeFreeThreadedBuildTest`, `IosFreeThreadedBuildTest`, `WasmGilBuildTest`). */ class DesktopFreeThreadedBuildTest { diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt index 4e7647f44..eeac2a85f 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/env/PublishedPythonVersionResourceTest.kt @@ -9,8 +9,9 @@ import kotlin.test.assertNotNull * Issue #61: the published jar carries the CPython version it embeds, agreeing with BuildConfig. * * Issue #158: the desktop jar embeds the free-threaded 3.15t, so its resource says - * `freeThreaded=true`. The AAR carries its own resource (`freeThreaded=false`, GIL until #328); - * the two are generated separately in `python-multiplatform/build.gradle.kts`. + * `freeThreaded=true`. The AAR carries its own resource, generated separately in + * `python-multiplatform/build.gradle.kts`; it says `freeThreaded=true` too since #328 + * (`AndroidFreeThreadedBuildTest.theAarSaysItIsFreeThreaded`). * * Red before #158: one resource served the jar and the AAR, rendered from * `-PpythonFreeThreaded` (default false), so the desktop jar said `freeThreaded=false`. diff --git a/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt index 475ae94f8..d39724875 100644 --- a/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt +++ b/python-multiplatform/src/iosMain/kotlin/python/multiplatform/EmbeddedPythonBuild.ios.kt @@ -1,4 +1,4 @@ package python.multiplatform -/** iOS embeds the GIL Python.xcframework until #328. See the `expect` in commonMain. */ -internal actual val embedsFreeThreadedPython: Boolean = false +/** iOS embeds the project's own free-threaded Python.xcframework, 3.15t (#328). See the `expect` in commonMain. */ +internal actual val embedsFreeThreadedPython: Boolean = true diff --git a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt new file mode 100644 index 000000000..4ec57ebf7 --- /dev/null +++ b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosFreeThreadedBuildTest.kt @@ -0,0 +1,33 @@ +package python.multiplatform + +import python.multiplatform.ffi.PythonTestFixture +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * SPEC T-2a (issue #328): iOS links the project's own free-threaded `Python.xcframework` (3.15t), + * the same archive the plugin's iOS consumer wiring stages. + * + * Red before the change: `embedsFreeThreadedPython` was `false` in `iosMain`, so `abiFlags` was `""` + * and `IosPythonHome`/`PythonHomeCheck` looked for `lib/python3.15`, and the library's cinterop and + * test binary used python.org's GIL framework (`Py_GIL_DISABLED` 0). (This replaces + * `IosGilBuildTest`, which pinned the GIL build until #328.) + */ +class IosFreeThreadedBuildTest { + + @Test + fun iosReportsTheFreeThreadedAbi() { + val v = Versions.currentVersion + assertEquals("t", v.abiFlags, "iOS embeds CPython ${v.compactVersionString}t (#328)") + assertEquals(v.compactVersionString + "t", v.taggedVersionString) + } + + @Test + fun theLinkedInterpreterIsReallyFreeThreaded() = PythonTestFixture.withInterpreter { + assertEquals( + "1", + PythonTestFixture.eval("__import__('sysconfig').get_config_var('Py_GIL_DISABLED')").toString(), + "the iOS framework is a GIL build; iOS must embed the free-threaded one (#328)", + ) + } +} diff --git a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt deleted file mode 100644 index 8c581259f..000000000 --- a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/IosGilBuildTest.kt +++ /dev/null @@ -1,22 +0,0 @@ -package python.multiplatform - -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * Issue #158 moved desktop to the free-threaded CPython 3.15t; iOS still embeds the GIL build until - * #328. This pins that the desktop change did not leak here. - * - * Not red before the change: iOS was GIL before and is GIL after. It is the guard against the - * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which - * `abiFlags` reads `"t"` here and the runtime looks for a `python3.15t` that iOS does not ship. - */ -class IosGilBuildTest { - - @Test - fun iosStillReportsTheGilAbi() { - val v = Versions.currentVersion - assertEquals("", v.abiFlags, "iOS embeds the GIL CPython until #328") - assertEquals(v.compactVersionString, v.taggedVersionString) - } -} diff --git a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt index 81b5fa356..258336660 100644 --- a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt +++ b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/ShapeDowncalls.kt @@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap * * `desktopMain` implements these over cached, unbound Panama `MethodHandle`s (one per * shape, see `Panama.kt`). `androidMain` implements these by delegating through - * JNI to Kotlin/Native trampolines compiled into `libmultiplatform_python3.13.so` (see + * JNI to Kotlin/Native trampolines compiled into `libmultiplatform_python3.15t.so` (see * `nativeMain/.../EmbedAPI.native.kt`), which invoke the target function pointer directly * via a `CFunction` cast -- no NDK, no C project, no ART entry-point patching. * diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 06c590c5b..391b3f21c 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -4,6 +4,7 @@ package python.multiplatform.ref import kotlinx.cinterop.* import python.multiplatform.currentThreadId +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.ProxyTypeFactory import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.PythonTestFixture @@ -173,21 +174,38 @@ class CycleCollectionTest { * proxy, and the number of proxies that were still owed if one stops doing so. */ private fun settleKotlinFinalisation(typeAddr: Long): Long { - val header = typeAddr.toCPointer() ?: return 0L - val started = header.pointed.value + if (typeAddr == 0L) return 0L + val started = typeRefCount(typeAddr) var last = started var quiet = 0 var turns = 0 while (quiet < 3 && turns < 50) { kotlin.native.runtime.GC.collect() platform.posix.usleep(2_000u) - val now = header.pointed.value + val now = typeRefCount(typeAddr) if (now == last) quiet++ else { quiet = 0; last = now } turns++ } return started - last } + /** + * `Py_REFCNT` of the object at [addr], for whichever object header this build has. On a GIL + * build `ob_refcnt` is the first field. On the free-threaded 3.15t, which every Kotlin/Native + * target here embeds (desktop #158, iOS and androidNative #328), offset 0 is `ob_tid` and the + * count is `ob_ref_local` (uint32 at +12) plus `ob_ref_shared` (at +16) shifted right by + * `_Py_REF_SHARED_SHIFT` (2). Same probe as the desktop `CycleCollectionTest.typeRefCount`. + * A plain, racy load, which is all the callers need. + */ + private fun typeRefCount(addr: Long): Long = + if (!embedsFreeThreadedPython) { + assertNotNull(addr.toCPointer()).pointed.value + } else { + val local = assertNotNull((addr + 12).toCPointer()).pointed.value.toLong() + val shared = assertNotNull((addr + 16).toCPointer()).pointed.value + local + (shared shr 2) + } + @Test fun testCycleCollectionByGC() = PythonTestFixture.withInterpreter { withGIL { @@ -320,8 +338,15 @@ class CycleCollectionTest { * releases it. Kotlin/Native's `proxyDealloc` hand-writes that (`PyObject_Type` for the * temporary, then two `Py_DecRef`s), so it can leak one type reference per instance by * forgetting, or free the type out from under the process by doing it twice. Both are checked - * by reading `ob_refcnt`, the first field of `PyObject` and fixed there by the stable ABI -- - * `sys.getrefcount` cannot be used because it counts its own argument's temporary. + * by reading the type's reference count out of its object header -- `sys.getrefcount` cannot + * be used because it counts its own argument's temporary. + * + * The header depends on the build. On a GIL build `ob_refcnt` is the first field. Every + * Kotlin/Native target here embeds the free-threaded 3.15t (desktop #158, iOS and androidNative + * #328), where offset 0 is `ob_tid` and the count is `ob_ref_local` (uint32 at +12) plus + * `ob_ref_shared` (at +16) shifted right by 2, and where a heap type's references are deferred + * until a collection materialises them, so one is taken on either side of the instantiation + * loop. Same probe and same reasoning as the desktop `CycleCollectionTest`. * * The "rises while alive" assertion is not decoration: it is what proves the probe reads a * real refcount, so the "returns afterwards" assertion cannot pass vacuously on an address @@ -334,12 +359,12 @@ class CycleCollectionTest { assertTrue(proxyTypeAddr != 0L, "Proxy type creation failed") val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - // The type object's own ob_refcnt, as a plain 64-bit load at the type's address. - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) - fun typeRefCount(): Long = typeHeader.pointed.value + // Free-threaded: materialise the deferred references the type already carries, so that + // `before` and `alive` are read in the same state. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount - val typeRefBefore = typeRefCount() + val typeRefBefore = typeRefCount(proxyTypeAddr) val rounds = 100 val proxies = LongArray(rounds) @@ -364,7 +389,10 @@ class CycleCollectionTest { liveBefore + rounds, HandleTable.liveCount, "setup failed: $rounds handles should be rooted while the proxies are alive", ) - val typeRefAlive = typeRefCount() + // The instantiations above were deferred on a free-threaded build; a collection is + // what turns them into countable references. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() + val typeRefAlive = typeRefCount(proxyTypeAddr) assertEquals( typeRefBefore + rounds, typeRefAlive, "each live instance of a heap type holds one reference to that type, so the count " + @@ -390,7 +418,7 @@ class CycleCollectionTest { "the table should be back to its starting size once every proxy is gone", ) assertEquals( - typeRefBefore, typeRefCount(), + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc must release the instance's reference to its heap type exactly once: " + "a count above $typeRefBefore means it was never released and the type leaks " + "per instance, below means it was released twice and the type will be freed " + @@ -440,10 +468,9 @@ class CycleCollectionTest { withGIL { val proxyTypeAddr = ProxyTypeFactory.createProxyType() val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) val liveBefore = HandleTable.liveCount - val typeRefBefore = typeHeader.pointed.value + val typeRefBefore = typeRefCount(proxyTypeAddr) val pyObjPtr = assertNotNull( PyObject_CallObject(proxyTypeAddr.toCPointer(), null), @@ -514,7 +541,7 @@ class CycleCollectionTest { ) assertEquals(liveBefore, HandleTable.liveCount, "the table did not return to its starting size") assertEquals( - typeRefBefore, typeHeader.pointed.value, + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc ran on a CPython-created thread but did not balance the instance's " + "reference to its heap type", ) diff --git a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt index 06a9dfc56..e634832d1 100644 --- a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt +++ b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/WasmGilBuildTest.kt @@ -4,8 +4,8 @@ import kotlin.test.Test import kotlin.test.assertEquals /** - * Issue #158 moved desktop to the free-threaded CPython 3.15t; wasmJs still embeds the GIL build until - * #175. This pins that the desktop change did not leak here. + * Issue #158 moved desktop to the free-threaded CPython 3.15t, and #328 moved Android, androidNative + * and iOS; wasmJs still embeds the GIL build until #175. This pins that neither change leaked here. * * Not red before the change: wasmJs was GIL before and is GIL after. It is the guard against the * naive implementation (flipping one shared `BuildConfig.pythonFreeThreaded`), under which From ba2973457fc41a09428ea77c3d14363597e32c50 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:31:00 +0900 Subject: [PATCH 45/61] Tests: assert 3.15t behaviour where four tests asserted GIL-build or 3.14t behaviour (#158) - PythonxCallableTest: sys.getrefcount includes the deferred-refcount bias (2**60) on 3.15t; read Long. - EvalCheckpointTest: 3.15.0 brc.c merges a detached owner's queue from the queuing thread (_PyThreadState_TrySuspendDetached), so cleaner decrefs can free before a checkpoint. - FreeThreadedGCGateTest: 3.15.0 removed gc_should_collect_mem_usage; process footprint no longer opens the GC gate. - nativeTest CycleCollectionTest: read the free-threaded object header (ob_ref_local + ob_ref_shared>>2). No product code changed. --- .../ffi/pythonx/PythonxCallableTest.kt | 13 ++- .../multiplatform/ref/EvalCheckpointTest.kt | 97 ++++++++++++++----- .../ref/FreeThreadedGCGateTest.kt | 86 +++++++++------- .../multiplatform/ref/CycleCollectionTest.kt | 64 +++++++++--- 4 files changed, 183 insertions(+), 77 deletions(-) diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt index 8140363d7..073deeefe 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt @@ -144,7 +144,12 @@ class PythonxCallableTest { _px_base = sys.getrefcount(_px_body) """.trimIndent(), ) - val base = eval("_px_base").toInt() + // Long, not Int. On the free-threaded build a module-level function is created with + // deferred reference counting (`func_new`: `_PyObject_SetDeferredRefcount` when the code is + // not nested), and `sys.getrefcount` reports the deferred bias with it: 2**60 + 1 on + // 3.15t, measured. That does not fit an Int. The bias is a constant, so every delta below + // still measures exactly the references the Kotlin side took and gave back. + val base = eval("_px_base").toLong() PythonCallables.withScope(scope) { pushComposer() @@ -160,19 +165,19 @@ class PythonxCallableTest { popComposer() } } - val held = eval("_px_held").toInt() + val held = eval("_px_held").toLong() assertTrue(held > base, "nothing on the Kotlin side took a reference: $base -> $held") assertEquals(1, scope.liveCount, "the scope did not record the callable it created") assertEquals(1, scope.close(), "closing the scope released nothing") Python3.exec("_px_released = sys.getrefcount(_px_body)") - assertEquals(base, eval("_px_released").toInt(), "the scope did not give the reference back") + assertEquals(base, eval("_px_released").toLong(), "the scope did not give the reference back") // The half a leak test does not cover. A second close must be a no-op, not a second decref: // one reference over-released here is a use-after-free somewhere unrelated later. assertEquals(0, scope.close(), "closing twice released something twice") Python3.exec("_px_twice = sys.getrefcount(_px_body)") - assertEquals(base, eval("_px_twice").toInt(), "a second close dropped the reference again") + assertEquals(base, eval("_px_twice").toLong(), "a second close dropped the reference again") } /** diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt index 59d2b236d..a0f6c8013 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ref/EvalCheckpointTest.kt @@ -21,10 +21,13 @@ import kotlin.time.measureTime * and friends therefore never merges the free-threaded build's biased reference-counting queue, * never processes QSBR-deferred frees and never runs a scheduled cyclic collection. * - * The consequence measured in §9 is that references handed back by the cleaner *stay* handed back - * and never freed: the counts are correct, the memory is not returned. These tests pin the shape - * of that, the fact that the obvious Stable ABI candidate does not fix it, and the cost of the one - * that does. + * The consequence measured in §9 on 3.14t is that references handed back by the cleaner *stay* + * handed back and never freed: the counts are correct, the memory is not returned. CPython 3.15 + * narrowed that: `_Py_brc_queue_object` (`Python/brc.c:118` at v3.15.0) now merges the owner's + * whole queue on its behalf when the owning thread is *detached*, which a Kotlin thread is between + * two `withGIL` scopes. What is left for the checkpoint is what was queued while the owner was + * attached. These tests pin the shape of that, the fact that the obvious Stable ABI candidate does + * not merge, and the cost of the one that does. */ class EvalCheckpointTest { @@ -125,8 +128,28 @@ class EvalCheckpointTest { * * where a reading exists that belongs to this test alone, the loop waits on that instead of * on the counter. With the global lock a released outer wrapper frees its list and hands the * target back a reference, so the target's own count says how many of *ours* the cleaner has - * reached. The free-threaded build has no such reading -- that it has none is the very thing - * the branch below asserts -- and there the drained counter is the evidence. + * reached. Free-threaded 3.15 has that reading too for every release that arrived while this + * thread was detached, but not for one that arrived while it was attached: that one sits in + * this thread's queue until a checkpoint, and the count never reaches the baseline on its + * own. So there the loop also stops once the cleaner has gone quiet, and the drain below is + * what has to finish the job. + * + * ### What the free-threaded branch asserts, and why it changed with 3.15 + * + * On 3.14t this branch asserted that the cleaner's $OUTER decrefs left the target's count + * exactly where it was: every one was queued to the owning thread. 3.15.0 added the detached + * owner case to `_Py_brc_queue_object` (`Python/brc.c:118`, `_PyThreadState_TrySuspendDetached`): + * if the owner is detached, the queuing thread suspends it, merges its whole queue and frees + * what reached zero. That is the state this thread is in while [collectorTest] waits, so the + * outer lists are now freed by the cleaner itself. What still holds, and is asserted: + * + * * the count dropped before any checkpoint was taken (the 3.15 merge exists; a return to the + * 3.14 queue-only behaviour reads `after the cleaner == after the build` and fails here); + * * `Py_MakePendingCalls`, inside one attached window so that no on-behalf merge can happen + * during it, does not move the count; + * * after [Python3.drainPendingReleases] every one of the $OUTER lists is gone, the same + * `<= refBefore` bound the global-lock branch is held to, which a drain that merged only + * part of the queue cannot pass. */ @Test fun testDrainPendingReleasesReclaimsWhatTheCleanerGaveBack(): CollectorTestResult { @@ -179,6 +202,11 @@ class EvalCheckpointTest { var attempts = 0 var cleanerReleases = 0 var refNow = refAfterBuild + // Free-threaded only: consecutive attempts in which the cleaner released nothing of + // its own, so that a straggler parked in this thread's queue cannot hold the loop open + // until the budget runs out. + var lastCleanerReleases = -1 + var cleanerQuietTurns = 0 // Driven rather than spun. On wasmJs the cleaner is a host finalisation callback that // arrives on a task, so a `while` loop here would ask 200 times inside one job and see @@ -207,10 +235,15 @@ class EvalCheckpointTest { refNow = refCount(target) attempts++ cleanerReleases = ReleaseCounter.released - releasedBefore - attempts * releasesPerProbe - // With the global lock, `refNow` is the reading that belongs to this test and - // to nothing else, so it is not enough for the counter alone to be satisfied. + if (cleanerReleases == lastCleanerReleases) cleanerQuietTurns++ + else { cleanerQuietTurns = 0; lastCleanerReleases = cleanerReleases } + // `refNow` is the reading that belongs to this test and to nothing else, so it + // is not enough for the counter alone to be satisfied. Free-threaded, a release + // that arrived while this thread was attached is queued, not freed, so a + // cleaner that has gone quiet ends the wait as well (see the KDoc). cleanerReleases >= OUTER && - (embedsFreeThreadedPython || refNow <= refBefore) + (refNow <= refBefore || + (embedsFreeThreadedPython && cleanerQuietTurns >= QUIET_TURNS_TO_SETTLE)) } }, finish = { try { @@ -235,24 +268,33 @@ class EvalCheckpointTest { val refAfterCleaner = refCount(target) if (embedsFreeThreadedPython) { - assertEquals( - refAfterBuild, - refAfterCleaner, - "free-threaded: a Py_DecRef from a thread that does not own the object is " + - "queued to its owner rather than run, so $OUTER completed decrefs must " + - "leave the count exactly where it was. If this now differs, the runtime " + - "has started merging somewhere else and the rest of this test is stale." + // 3.15's on-behalf merge (Python/brc.c:118): this thread was detached for + // almost all of the wait, so the cleaner itself freed the outer lists. + assertTrue( + refAfterCleaner < refAfterBuild, + "free-threaded 3.15: a Py_DecRef from another thread while the owner is " + + "detached merges the owner's queue on its behalf, so the cleaner " + + "alone should have freed outer lists and lowered the count. It did " + + "not move (before the build: $refBefore, after it: $refAfterBuild, " + + "after the cleaner: $refAfterCleaner; $diagnosis): the runtime is back " + + "to queuing every cross-thread release for the owner's next checkpoint" ) - // Py_MakePendingCalls is the Stable ABI function that looks like it should do - // this. It does not: it handles _PY_CALLS_TO_DO_BIT and _PY_SIGNALS_PENDING_BIT - // and returns, while the merge hangs off _PY_EVAL_EXPLICIT_MERGE_BIT, which only - // _Py_HandlePending clears. - val pendingResult = withGIL { Py_MakePendingCalls() } + // Py_MakePendingCalls is the Stable ABI function that looks like it should + // merge. It does not: it handles _PY_CALLS_TO_DO_BIT and + // _PY_SIGNALS_PENDING_BIT and returns, while the merge hangs off + // _PY_EVAL_EXPLICIT_MERGE_BIT, which only _Py_HandlePending clears. All + // three steps run in one attached window: while this thread is attached no + // other thread can merge its queue for it, so a change here is the call's. + val (beforePending, pendingResult, afterPending) = withGIL { + val a = refCount(target) + val r = Py_MakePendingCalls() + Triple(a, r, refCount(target)) + } assertEquals(0, pendingResult, "Py_MakePendingCalls should not have failed") assertEquals( - refAfterBuild, - refCount(target), + beforePending, + afterPending, "Py_MakePendingCalls must not be mistaken for a checkpoint: it does not merge " + "the biased reference-counting queue" ) @@ -260,9 +302,12 @@ class EvalCheckpointTest { Python3.drainPendingReleases() val refAfterDrain = refCount(target) assertTrue( - refAfterDrain < refAfterBuild, - "reaching an eval-loop checkpoint should have merged all $OUTER queued " + - "releases (before: $refAfterBuild, after: $refAfterDrain)" + refAfterDrain <= refBefore, + "after an eval-loop checkpoint every one of the $OUTER outer lists should " + + "be gone, the ones the cleaner freed and the ones queued while this " + + "thread was attached (before the build: $refBefore, after it: " + + "$refAfterBuild, after the cleaner: $refAfterCleaner, after the " + + "drain: $refAfterDrain; $diagnosis)" ) } else { // `<= refBefore`, not merely `< refAfterBuild`: every one of the $OUTER diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt index 38b439ec4..691c43e70 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ref/FreeThreadedGCGateTest.kt @@ -34,6 +34,18 @@ import kotlin.test.assertTrue * Both tests below are matched pairs: each manipulates something that the free-threaded source * says is load-bearing and the global-lock source says is irrelevant, and asserts exactly that * split. If CPython changes either gate, they fail and say which one. + * + * ### CPython 3.15 removed the process-memory gate + * + * Desktop embeds 3.15.0t since issue #158, and [freeThreadedSchedulingTracksWholeProcessFootprintNotThePythonHeap] + * did what it was written to do: it failed when the gate changed. At the v3.15.0 tag + * `gc_should_collect_mem_usage`, `get_process_mem_usage`, `last_mem` and `deferred_count` are gone + * (`Python/gc_free_threading.c`, `Include/internal/pycore_interp_structs.h`), and + * `gc_should_collect` (`:1996`) returns true once `young.count` clears the threshold and + * `long_lived_total / 4` (`:2009`). The `old[0].threshold == 0` short circuit is still there + * (`:2004`), and `gc_collect_main` still re-asks `gc_should_collect` for a heap-triggered + * collection (`:2217`). The table above is the 3.14.7 record; the first test's assertions hold on + * both, the second asserts the 3.15 shape. */ class FreeThreadedGCGateTest { @@ -57,10 +69,10 @@ class FreeThreadedGCGateTest { /** * `gc.collect()`. * - * Free-threaded this also **resets `last_mem` to the current process footprint** - * (`gc_free_threading.c:2301`, reached because `_Py_GC_REASON_MANUAL` skips the gate at - * `:2328`), which is what makes the arms below independent of each other: every arm starts - * from a freshly pinned memory baseline. + * It resets `young.count` and sets `long_lived_total` to what survived, which is what makes + * the arms below independent of each other: every arm starts from the same scheduling state. + * On 3.14t it also reset `last_mem` to the current process footprint (`gc_free_threading.c:2301` + * at 3.14.7), the memory baseline the second test's 3.14 form depended on. */ private fun forceCollect(gc: PyObject) { val collect = gc.getAttr("collect") @@ -69,7 +81,8 @@ class FreeThreadedGCGateTest { } /** `gc.get_stats()[0]['collections']` -- incremented only *past* the gate at - * `gc_free_threading.c:2328`, so it counts collections that actually ran, not bits set. */ + * `gc_free_threading.c:2217` (v3.15.0; `:2328` at 3.14.7), so it counts collections that + * actually ran, not bits set. */ private fun gen0Collections(gc: PyObject): Long { val f = gc.getAttr("get_stats") val t = f.invoke() @@ -124,7 +137,7 @@ class FreeThreadedGCGateTest { * `gc.set_threshold(_, 0, _)` -- a setting about **generation 1** -- is what decides whether * **generation 0** is collectable at all on the free-threaded build. * - * `gc_free_threading.c:2126` short-circuits `gc_should_collect` to `true` when + * `gc_free_threading.c:2004` (v3.15.0; `:2126` at 3.14.7) short-circuits `gc_should_collect` to `true` when * `old[0].threshold == 0`, skipping both `long_lived_total/4` and the process-memory gate. * Nothing in `gc.c` reads that field while scheduling, so the same change must be a no-op with * the global lock. Run A/B/A in one JVM so the §8b latch cannot account for the difference. @@ -152,13 +165,13 @@ class FreeThreadedGCGateTest { setThreshold(2000, 10, 10) val defaultAfter = arm(gc, builtins, "default-thresholds-again") - // Holds on both builds, for different reasons: free-threaded because :2126 + // Holds on both builds, for different reasons: free-threaded because :2004 // skips the gates, GIL-side because there was never a gate to skip. assertTrue( bypassed < reclaimedBound, "with gc.set_threshold(2000, 0, 0) the eval-loop checkpoint should reclaim " + "the bulk of $totalObjects dropped objects, but $bypassed remained. " + - "Free-threaded that means gc_free_threading.c:2126 no longer " + + "Free-threaded that means gc_free_threading.c:2004 (v3.15.0) no longer " + "short-circuits gc_should_collect on old[0].threshold == 0" ) @@ -183,27 +196,33 @@ class FreeThreadedGCGateTest { } /** - * Free-threaded, whether the Python cyclic collector runs is a function of the **JVM's** - * memory, not of Python's. + * Whether the Python cyclic collector runs must not depend on the **JVM's** memory. * - * `gc_should_collect_mem_usage` (`gc_free_threading.c:2080`) asks whether the *whole process* - * footprint has grown by more than `last_mem / 10` since the last collection. So faulting in - * 400 MB of ballast partway through the workload -- changing no threshold, allocating no extra - * Python container, touching nothing the collector tracks -- should open the gate, and - * dropping the ballast should shut it again. `gc.c` has no such gate, so the identical - * manipulation must be a no-op with the global lock. + * Faulting in 400 MB of ballast partway through the workload changes no threshold, allocates + * no extra Python container and touches nothing the collector tracks. On 3.14t that opened + * `gc_should_collect_mem_usage` (`gc_free_threading.c:2080` at 3.14.7), which asked whether + * the *whole process* footprint had grown by more than `last_mem / 10` since the last + * collection. Measured on 3.14t (3 consecutive JVMs, identical to the object), residue out of + * 20,000: * - * Measured (3 consecutive JVMs, identical to the object), residue out of 20,000: - * - * | arm | free-threaded | global lock | + * | arm | free-threaded 3.14t | global lock | * |---|---:|---:| * | baseline | 20,000 (0 collections) | 1,972 (9 collections) | * | 400 MB faulted in midway | **9,754 (1 collection)** | 1,988 (9 collections) | * | ballast dropped | 20,000 (0 collections) | 1,968 (9 collections) | * - * The 400 MB figure assumes the test JVM's footprint is under ~4 GB, so that 400 MB clears the - * `last_mem / 10` bar; the failure message says so, because a much larger heap would make this - * test wrong rather than the mechanism wrong. + * 3.15.0 deleted that gate (see the class KDoc), so on the 3.15t desktop embeds the ballast + * must be a no-op, as it always was with the global lock. Each arm starts from a forced + * collection and runs the identical workload, so the scheduling inputs that remain + * (`young.count`, the threshold, `long_lived_total`) are the same in all three and so is the + * outcome. What is asserted free-threaded is exactly that: the ballast arm and the + * ballast-dropped arm land within a quarter of the workload of the baseline. The bound is not + * "everything was reclaimed", because whether `long_lived_total / 4` lets this process collect + * at all depends on how many objects the test JVM's interpreter already holds, which this test + * does not control. + * + * What it still catches: a process-memory gate coming back. The 3.14t row above differs from + * its own baseline by 10,246 objects, more than twice the 5,000 allowed. */ @Test fun freeThreadedSchedulingTracksWholeProcessFootprintNotThePythonHeap() = @@ -221,18 +240,19 @@ class FreeThreadedGCGateTest { val dropped = arm(gc, builtins, "ballast-dropped") if (embedsFreeThreadedPython) { - // The whole finding: JVM memory growth, and nothing else, collected Python - // cycles. Deliberately not a tight bound -- how much a single collection - // takes depends on where in the workload the footprint crossed the bar. + // 3.15: no process-memory input, so the ballast must not change the + // outcome. A quarter of the workload absorbs a collection landing a few + // objects earlier or later in one arm; it does not absorb 3.14t's 10,246. + val tolerance = totalObjects / 4 assertTrue( - withBallast < baseline, - "faulting in 400 MB of process memory partway through the workload " + - "should have opened gc_should_collect_mem_usage " + - "(gc_free_threading.c:2100) and let the checkpoint collect, but " + - "the ballast arm left $withBallast against the baseline's " + - "$baseline out of $totalObjects. Either that gate is gone, or " + - "this JVM's footprint is large enough (>~4 GB) that 400 MB no " + - "longer clears last_mem/10" + kotlin.math.abs(withBallast - baseline) < tolerance && + kotlin.math.abs(dropped - baseline) < tolerance, + "free-threaded 3.15.0 has no process-memory gate (gc_should_collect, " + + "gc_free_threading.c:1996), so faulting in and dropping 400 MB of " + + "process memory should not change what the checkpoint collects, but " + + "the arms left $baseline / $withBallast / $dropped out of " + + "$totalObjects (allowed spread: $tolerance). A gate on whole-process " + + "memory, like 3.14's gc_should_collect_mem_usage, is back" ) } else { // gc.c never calls get_process_mem_usage and has no last_mem field at all. diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt index 06c590c5b..aa7b1f65a 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ref/CycleCollectionTest.kt @@ -4,6 +4,7 @@ package python.multiplatform.ref import kotlinx.cinterop.* import python.multiplatform.currentThreadId +import python.multiplatform.embedsFreeThreadedPython import python.multiplatform.ffi.ProxyTypeFactory import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.PythonTestFixture @@ -173,21 +174,38 @@ class CycleCollectionTest { * proxy, and the number of proxies that were still owed if one stops doing so. */ private fun settleKotlinFinalisation(typeAddr: Long): Long { - val header = typeAddr.toCPointer() ?: return 0L - val started = header.pointed.value + if (typeAddr == 0L) return 0L + val started = typeRefCount(typeAddr) var last = started var quiet = 0 var turns = 0 while (quiet < 3 && turns < 50) { kotlin.native.runtime.GC.collect() platform.posix.usleep(2_000u) - val now = header.pointed.value + val now = typeRefCount(typeAddr) if (now == last) quiet++ else { quiet = 0; last = now } turns++ } return started - last } + /** + * `Py_REFCNT` of the object at [addr], for whichever object header this build has. On a GIL + * build `ob_refcnt` is the first field. On the free-threaded 3.15t, which desktop Kotlin/Native + * embeds since #158 (iOS and androidNative from #328), offset 0 is `ob_tid` and the + * count is `ob_ref_local` (uint32 at +12) plus `ob_ref_shared` (at +16) shifted right by + * `_Py_REF_SHARED_SHIFT` (2). Same probe as the desktop `CycleCollectionTest.typeRefCount`. + * A plain, racy load, which is all the callers need. + */ + private fun typeRefCount(addr: Long): Long = + if (!embedsFreeThreadedPython) { + assertNotNull(addr.toCPointer()).pointed.value + } else { + val local = assertNotNull((addr + 12).toCPointer()).pointed.value.toLong() + val shared = assertNotNull((addr + 16).toCPointer()).pointed.value + local + (shared shr 2) + } + @Test fun testCycleCollectionByGC() = PythonTestFixture.withInterpreter { withGIL { @@ -320,8 +338,15 @@ class CycleCollectionTest { * releases it. Kotlin/Native's `proxyDealloc` hand-writes that (`PyObject_Type` for the * temporary, then two `Py_DecRef`s), so it can leak one type reference per instance by * forgetting, or free the type out from under the process by doing it twice. Both are checked - * by reading `ob_refcnt`, the first field of `PyObject` and fixed there by the stable ABI -- - * `sys.getrefcount` cannot be used because it counts its own argument's temporary. + * by reading the type's reference count out of its object header -- `sys.getrefcount` cannot + * be used because it counts its own argument's temporary. + * + * The header depends on the build. On a GIL build `ob_refcnt` is the first field. Desktop + * Kotlin/Native embeds the free-threaded 3.15t since #158 (iOS and androidNative from #328), + * where offset 0 is `ob_tid` and the count is `ob_ref_local` (uint32 at +12) plus + * `ob_ref_shared` (at +16) shifted right by 2, and where a heap type's references are deferred + * until a collection materialises them, so one is taken on either side of the instantiation + * loop. Same probe and same reasoning as the desktop `CycleCollectionTest`. * * The "rises while alive" assertion is not decoration: it is what proves the probe reads a * real refcount, so the "returns afterwards" assertion cannot pass vacuously on an address @@ -334,12 +359,12 @@ class CycleCollectionTest { assertTrue(proxyTypeAddr != 0L, "Proxy type creation failed") val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - // The type object's own ob_refcnt, as a plain 64-bit load at the type's address. - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) - fun typeRefCount(): Long = typeHeader.pointed.value + // Free-threaded: materialise the deferred references the type already carries, so that + // `before` and `alive` are read in the same state. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount - val typeRefBefore = typeRefCount() + val typeRefBefore = typeRefCount(proxyTypeAddr) val rounds = 100 val proxies = LongArray(rounds) @@ -364,7 +389,10 @@ class CycleCollectionTest { liveBefore + rounds, HandleTable.liveCount, "setup failed: $rounds handles should be rooted while the proxies are alive", ) - val typeRefAlive = typeRefCount() + // The instantiations above were deferred on a free-threaded build; a collection is + // what turns them into countable references. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() + val typeRefAlive = typeRefCount(proxyTypeAddr) assertEquals( typeRefBefore + rounds, typeRefAlive, "each live instance of a heap type holds one reference to that type, so the count " + @@ -390,7 +418,7 @@ class CycleCollectionTest { "the table should be back to its starting size once every proxy is gone", ) assertEquals( - typeRefBefore, typeRefCount(), + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc must release the instance's reference to its heap type exactly once: " + "a count above $typeRefBefore means it was never released and the type leaks " + "per instance, below means it was released twice and the type will be freed " + @@ -440,10 +468,18 @@ class CycleCollectionTest { withGIL { val proxyTypeAddr = ProxyTypeFactory.createProxyType() val proxyType = assertNotNull(proxyTypeAddr.toCPointer()) - val typeHeader = assertNotNull(proxyTypeAddr.toCPointer()) + + // Free-threaded: an instance takes its type reference through this thread's per-thread + // refcount array (`_Py_INCREF_TYPE`), while proxyDealloc gives it back with a header + // `Py_DecRef`, so the type is cached across tests and earlier instantiations on this + // thread can still be unmerged here. The worker's gc.collect() below merges every + // thread's array (`gc_free_threading.c:2090` at v3.15.0); collecting first puts the + // opening reading in that same merged state, so the comparison is instantiation and + // dealloc and nothing left over. + if (embedsFreeThreadedPython) python.native.ffi.PyGC_Collect() val liveBefore = HandleTable.liveCount - val typeRefBefore = typeHeader.pointed.value + val typeRefBefore = typeRefCount(proxyTypeAddr) val pyObjPtr = assertNotNull( PyObject_CallObject(proxyTypeAddr.toCPointer(), null), @@ -514,7 +550,7 @@ class CycleCollectionTest { ) assertEquals(liveBefore, HandleTable.liveCount, "the table did not return to its starting size") assertEquals( - typeRefBefore, typeHeader.pointed.value, + typeRefBefore, typeRefCount(proxyTypeAddr), "tp_dealloc ran on a CPython-created thread but did not balance the instance's " + "reference to its heap type", ) From 7c39eba1f7e5d357871df645d4b6869204c6c9ea Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:29:29 +0900 Subject: [PATCH 46/61] Docs: record the first 3.15t desktop CI run and what changed from 3.14t (#158) ROADMAP 9 gets the counts of the first free-threaded 3.15t runs (desktop 647 tests, linuxX64 497) and the four 3.14t-only assertions corrected in #398. gc-scheduling-investigation sections 2 and 8d get 3.15.0 notes: brc.c:118 merges a detached owner's queue; the GC process-memory gate is removed. --- docs/investigations/gc-scheduling-investigation.md | 13 +++++++++++++ docs/roadmap/ROADMAP.md | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/investigations/gc-scheduling-investigation.md b/docs/investigations/gc-scheduling-investigation.md index 5be003937..7da16ffcb 100644 --- a/docs/investigations/gc-scheduling-investigation.md +++ b/docs/investigations/gc-scheduling-investigation.md @@ -60,6 +60,13 @@ return and that the refcount did not move). ## 2. Free-threading: biased reference counting, and the queue only the eval loop drains +> **Update for 3.15.0 (2026-11-13, #398).** This section describes 3.14.7. In CPython 3.15.0, +> `_Py_brc_queue_object` gained a second path (`Python/brc.c:118`, `_PyThreadState_TrySuspendDetached`): +> when the owning thread is detached, the queuing thread merges the owner's queue on its behalf and frees +> what reaches zero. An embedder's thread is detached between `withGIL` scopes, so on 3.15t a cleaner's +> releases can be reclaimed without the owner reaching the eval loop. `EvalCheckpointTest` asserts the +> 3.15.0 behaviour. Whether this lowers the need for `autoDrainInterval` (ROADMAP 9.1) is not measured. + On a free-threaded build `Py_DECREF` branches on whether the thread owns the object (`ob_tid`): owner decrements `ob_ref_local` and may `tp_dealloc` at once; a non-owner decrements `ob_ref_shared` atomically, and if that takes it to zero the object is queued to its owner by @@ -188,6 +195,12 @@ the finding. ### 8d. Answered: the checkpoint is fine, the collector asks a second question +> **Update for 3.15.0 (2026-11-13, #398).** The table below is v3.14.7. In v3.15.0 the process-memory gate +> is gone (`gc_should_collect_mem_usage`, `get_process_mem_usage`, `last_mem` and `deferred_count` were +> removed): `gc_should_collect` (`Python/gc_free_threading.c:1996`) checks only the young threshold, +> `old[0].threshold == 0` (`:2004`) and `long_lived_total / 4` (`:2009`). `FreeThreadedGCGateTest` asserts +> that process footprint no longer moves the gate. The 8a to 8c measurements were not repeated on 3.15t. + Source-confirmed asymmetry (v3.14.7): | | GIL (`gc.c`) | free-threaded (`gc_free_threading.c`) | diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index 547a8f162..4de133a42 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -165,9 +165,13 @@ Desktop (JVM, `macosArm64`, `linuxX64`) embeds the free-threaded CPython 3.15t a kept as an option. Baseline is CPython 3.15t"; leader, 2026-11-13: "flip the desktop default to 3.15t now, before the #377 alpha03 release"). `-PpythonFreeThreaded` is removed; the build fails if it is passed. Android and iOS still embed the GIL 3.15 (9.2), wasm GIL (#175); the per-platform fact is -`embedsFreeThreadedPython` (SPEC T-1). The 3.15t default has not yet been observed running the desktop -suite; the last recorded free-threaded run is 3.14t as an opt-in: 236 tests, 0 failures, 1 skipped, the -same count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is +`embedsFreeThreadedPython` (SPEC T-1). The 3.15t default first ran the suites in CI on 2026-11-13: Desktop CI +run 43306301208 counted `desktopTest` 647 tests, 3 failed, 1 skipped, and `linuxX64Test` 497 tests, 3 failed. +All six failures were tests asserting GIL-build or 3.14t behaviour (a `sys.getrefcount` that includes the +deferred-refcount bias, 3.15.0's detached-owner merge in `brc.c`, the removed GC memory gate, a probe +reading `ob_tid` as the refcount). They were corrected in #398 with no product change, and run 43329515729 +passed (Gradle prints no count on success). The earlier free-threaded record is 3.14t as an opt-in: 236 +tests, 0 failures, 1 skipped (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is drained by the eval-loop checkpoint (`PM/commonTest/.../ref/EvalCheckpointTest.kt`, `docs/investigations/gc-scheduling-investigation.md`); `FreeThreadedGCGateTest` guards the GC probe. The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done From 1ff3e3e5dd3cb26b6614855c748dab4257b20972 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:23:21 +0900 Subject: [PATCH 47/61] TypedPython test: count a probe's late success in the buffer size checks (#401) probe_during_call ignored a probe that succeeded after the compiled call returned, but its effect stayed (one more element after a late append), so len(a) == N failed on macOS. The probe now reports late successes and the two size checks add them back. A success during the call still fails the test. --- .../src/test/python/test_buffer_memoryview.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py index e622536b9..c0862bed1 100644 --- a/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py +++ b/python-multiplatform-ksp/src/test/python/test_buffer_memoryview.py @@ -480,11 +480,13 @@ def calibrated_spin(built): def probe_during_call(call, probes, attempts=3): """Run call() on the main thread while a worker applies each probe in a loop. Returns - (result, {probe_name: (BufferError count, other outcomes)}). Retries `attempts` times when a - probe never observed the call (machine too fast or too loaded).""" + (result, {probe_name: (BufferError count, other outcomes, late successes)}). Retries `attempts` + times when a probe never observed the call (machine too fast or too loaded). A late success is a + probe that succeeded only after the call returned: it is not a failure, but its effect stays (an + append left one more element), so callers that check sizes add it back (#401).""" for _ in range(attempts): go, done = threading.Event(), threading.Event() - seen = {name: [0, []] for name in probes} + seen = {name: [0, [], 0] for name in probes} def worker(): go.wait() @@ -504,6 +506,8 @@ def worker(): # stays unset for far longer than this grace period. if not done.wait(0.05): seen[name][1].append("succeeded") + else: + seen[name][2] += 1 return time.sleep(0.001) @@ -533,7 +537,7 @@ def test_other_thread_release_and_append_get_bufferror_during_a_memoryview_call( assert result == float(n) assert seen["release"][0] > 0 and seen["release"][1] == [], seen assert seen["append"][0] > 0 and seen["append"][1] == [], seen - assert len(a) == N # nothing resized under the call + assert len(a) == N + seen["append"][2] # nothing resized under the call (#401) assert release_ok(v) and can_resize(a) # held only during the call @@ -547,7 +551,8 @@ def test_other_thread_append_gets_bufferror_during_an_array_call(built): assert result == float(n) for name in ("append", "extend", "pop"): assert seen[name][0] > 0 and seen[name][1] == [], (name, seen) - assert len(a) == N + # nothing resized under the call; a probe that succeeded after it returned stays applied (#401) + assert len(a) == N + seen["append"][2] + seen["extend"][2] - seen["pop"][2] assert can_resize(a) From 5af19ab674bf9e7ee75f0c77e727172b96dae585 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:40:29 +0900 Subject: [PATCH 48/61] CI: run the Gradle plugin's unit tests in desktop.yml (#403) New job on ubuntu-latest: :python-multiplatform-gradle-plugin:test --rerun, with the test count in the job summary and the reports uploaded on failure. Leader decision 2026-11-13. --- .github/workflows/desktop.yml | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 907744729..3c741feb9 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -51,6 +51,54 @@ jobs: # Using --console=plain so logs are readable in GitHub Actions run: ./gradlew :python-multiplatform:desktopTest --console=plain + plugin-unit: + # The Gradle plugin's own unit tests (#403). Only its fixtures ran in CI before, so plugin + # unit tests were never executed (leader decision 2026-11-13). Linux: the tests use fake + # trees and need no Xcode or simulator. + name: Gradle plugin unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'zulu' + cache: 'gradle' + + - name: Run the plugin's unit tests + # --rerun: never read an earlier result as this run's (AGENTS.md rule 8). + run: ./gradlew :python-multiplatform-gradle-plugin:test --rerun --console=plain + + - name: Count the tests + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import glob, xml.etree.ElementTree as ET + files = glob.glob("python-multiplatform-gradle-plugin/build/test-results/test/*.xml") + total = failed = skipped = 0 + for f in files: + suite = ET.parse(f).getroot() + total += int(suite.get("tests", 0)) + failed += int(suite.get("failures", 0)) + int(suite.get("errors", 0)) + skipped += int(suite.get("skipped", 0)) + print(f"Gradle plugin unit tests: {total} tests, {total - failed - skipped} passed, {failed} failed, {skipped} skipped ({len(files)} suites)") + if not files: + raise SystemExit("no plugin test results were written") + PY + cat "$GITHUB_STEP_SUMMARY" + + - name: Upload the test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gradle-plugin-unit-reports + path: | + python-multiplatform-gradle-plugin/build/reports/tests/test + python-multiplatform-gradle-plugin/build/test-results/test + desktop-native-linux: name: Desktop Kotlin/Native Tests (Linux x86_64) runs-on: ubuntu-latest From 1ae08b1cd9d258fdb6510b36f2eeca0164e1bc86 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:00:03 +0900 Subject: [PATCH 49/61] TypedPython: move New's argument references into the slots; borrow local operands (#125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Semantics-preserving allocation work under the owner's standing intent (no GIL-only features, no observable change; leader, 2026-11-13). tp_new_fixed_move hands the argument temporaries' references to the fresh object instead of copying and releasing them; Is, IsExact, FieldGet and C-to-C Call arguments borrow a plain OBJ local's own reference. Removes +1/-1 pairs with nothing in between that can run code. Design 4.3.4 바 lists the other candidates and why they are not done. --- docs/design/typedpython.md | 110 ++++ .../src/main/python/typedpython/cgen.py | 87 +++- .../main/python/typedpython/runtime/API.md | 4 + .../python/typedpython/runtime/tp_runtime.h | 40 ++ .../src/test/python/test_alloc_preserving.py | 476 ++++++++++++++++++ 5 files changed, 699 insertions(+), 18 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_alloc_preserving.py diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 686ebc722..7540b1685 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -410,6 +410,116 @@ PyPy·GraalVM 과 같은 방식이다. N-11 클래스의 객체를 PyObject 대 이 셋은 N-12 이전부터 있는 경계도 포함한다. 1번이 결정되어 N-12 구현을 시작한다. 2번은 #141(1단계 완료), 3번은 #142. +#### 바. 관측을 바꾸지 않는 할당·해제·refcount 비용 (#125, 조사 2026-11-13) + +**제약 (리더 경유, 2026-11-13, 오너의 상시 의도):** "All three GC options listed before are REJECTED: +per-type freelist (replaces tp_dealloc, id reuse), deferring or skipping GC tracking (gc.is_tracked / +gc.get_objects observable), batching GC allocation counts or adjusting thresholds (changes when +collections run). The rule behind it: no GIL-only features (everything must work on free-threaded +3.15t), and no change to any observable Python behaviour (gc module, ids, weakrefs, __del__ timing, +collection timing, sys.getrefcount, tracemalloc, subclassing, later class changes)." + +> 이 제약의 "collection timing" 은 마.1 의 결정 (2026-10-04, 사용자: GC 수집 시점은 동일성 범위에서 뺀다)보다 +> 좁다. 이 절의 후보는 둘 다 만족하도록 골랐다(수집 시점을 바꾸는 후보는 모두 뺐다). 두 문장 중 어느 쪽이 +> N-12a/b 의 가상 객체에도 적용되는지는 이 조사가 정하지 않는다(리더에게 확인할 점). + +**출발점.** 프로파일(`benchmarks/typedpython/results/2026-10-25-binary-trees-alloc-profile.md`, 3.15.0rc1t, +macOS arm64)은 #273 과 #371 이전 것이다. 몫은 그때의 전체에 대한 비율이고, 아래의 기대 이득은 그 몫에서 어림한 +**추정**이다(측정 아님). 컴파일 함수는 `make`, `check` 그대로다(`test_class_methods.py` EXPECTED). +CPython 출처는 v3.15.0rc1 태그의 파일과 함수 이름이다. + +**후보별 결론.** + +1. **같은 할당 경로를 단계만 줄이기.** `PyType_GenericAlloc` (`Objects/typeobject.c`) 은 + `_PyType_AllocNoTrack` 에서 크기 계산(`_PyObject_VAR_SIZE(type, nitems+1)`, itemsize 0 이면 basicsize), + `_PyObject_MallocWithType`, `_PyObject_GC_Link` (= `record_allocation(tstate)`, 할당 계수), 헤더 뒤 전체 + `memset`, `_PyObject_Init` (3.15t 에서는 heap type 의 참조를 `_Py_INCREF_TYPE`, 즉 스레드별 표에 센다, + `Include/internal/pycore_object.h`) 을 하고 마지막에 `_PyObject_GC_TRACK` 한다. 고정 레이아웃 정확 타입에서 + 남는 중복은 `memset` 하나다(모든 슬롯을 곧 쓴다). 공개 API `PyObject_GC_New` + `PyObject_GC_Track` 은 + `gc_alloc` (`Python/gc_free_threading.c`) 로 같은 할당기·같은 크기·같은 `record_allocation`·같은 + `_PyObject_Init` 를 거치고 `memset` 만 없다. 그래서 gc, tracemalloc, 할당 계수 어느 것도 다르지 않다. + 다만 객체의 모든 바이트를 컴파일 코드가 써야 하므로 클래스의 모든 슬롯이 필드여야 한다. 기대 이득: + memset 17 샘플, 약 1% (추정). **하지 않음:** 이득이 작다. 직접 할당(`tp_alloc` 를 건너뛰고 헤더를 손으로 + 채우기)은 비공개 API 이고, `Py_INCREF(type)` 로 바꾸면 3.15t 의 스레드별 type refcount 와 달라져 + `sys.getrefcount(Node)` 의 회계가 바뀌므로 기각. + 할당 하나에 CPython 안에서 스레드 상태 읽기가 셋 이상 있다(`gc_alloc`/`_PyObject_MallocWithType` 의 힙, + `record_allocation`, `_Py_INCREF_TYPE`), 해제 하나에도 셋(`_Py_Dealloc` 의 trashcan 여유 확인, + `PyObject_GC_Del` 의 `record_deallocation`, `_Py_DECREF_TYPE`). tstate 를 받는 공개 변형이 없으므로 공개 + API 로는 줄일 수 없다. 프로파일의 `_tlv_get_addr` 몫은 macOS 의 TLS 접근 방식 때문에 큰 것으로 보인다(추정). +2. **스레드 지역 객체의 refcount 피하기(3.15t biased refcounting).** `Py_INCREF` (`Include/refcount.h`) 는 + 3.15t 에서 `ob_ref_local` 을 relaxed 로 읽고, immortal 이면 끝, 소유 스레드(`ob_tid == _Py_ThreadId()`, + arm64 에서는 레지스터 읽기)면 원자 연산 없이 저장, 아니면 `ob_ref_shared` 에 원자 덧셈이다. `Py_DECREF` 도 + 같고, 0 이 되면 `_Py_MergeZeroLocalRefcount` 가 `ob_ref_shared == 0` 이면 바로 `_Py_Dealloc` 한다. 이 스레드가 + 만든 노드는 이미 싼 경로를 탄다. 그래서 "스레드 지역이면 refcount 를 다르게" 는 연산 하나당 더 얻을 것이 + 없다. 얻을 것은 **증감 쌍 자체를 없애는 것**이고, 그 조건은 스레드 지역성이 아니라 "다른 참조가 그동안 + 살려 둔다" 이다(아래 구현). 스레드 지역 증명은 슬롯 값을 빌려 읽을 때만 필요하다(나., D5, 바뀌지 않음). +3. **임시 객체 없애기(N-12a 가상 객체).** binary-trees 가 이득을 못 보는 이유: `make` 의 객체는 반환으로 + 탈출하고, `check` 는 아무것도 할당하지 않는다. 두 함수가 만나는 곳은 `check(make(d))` 를 부르는 `main` + 뿐인데, `main` 은 비용 정책(#150)으로 컴파일되지 않고 순수하지도 않다(`print`, `sys.argv`). 그래서 N-12b + 영역의 host 가 없고(별도 벤치 `binary_trees_region.py` 가 그 경우를 잰다), `make` 와 `check` 를 합치거나 + (`check∘make` 를 트리 없이 세기) `check` 를 호출자에 inline 하는 것도 호출 지점이 인터프리터 안이라 할 + 수 없다. `main` 을 컴파일하면 벤치의 컴파일 함수 집합이 바뀌므로 이 조사의 범위 밖이다. +4. **해제를 타입의 tp_dealloc 그대로, 간접 호출만 줄이기.** 지금 경로: `Py_DECREF` → + `_Py_MergeZeroLocalRefcount` → `_Py_Dealloc` (`Objects/object.c`: tstate 읽기, 재귀 여유가 작으면 trashcan, + reftracer) → `subtype_dealloc` (`PyObject_GC_UnTrack`, 베이스 사슬, finalizer·weaklist 확인, `clear_slots` + 가 자식마다 `Py_DECREF`, 재추적) → `object_dealloc` → `tp_free = PyObject_GC_Del` → `_Py_DECREF_TYPE`. 간접 + 호출은 둘셋이고 비용은 그 안의 일이다. 같은 일을 컴파일 코드가 하려면 비공개 API(`_PyTrash_*`, + `_Py_DECREF_TYPE`)가 필요하고 `tp_dealloc` 를 우회하게 되어 2026-10-25 결정(미룸)의 문제와 같다. 해제 + 순서를 바꾸면(예: 재귀 대신 반복) 이후 할당의 주소 재사용, 곧 `id()` 값이 달라진다. **하지 않음.** +5. **스레드 상태 몫 가운데 #273 이 남긴 것.** 컴파일 프레임마다 `tp_enter_call` 이 `PyThreadState_GetFrame` + (`Python/pystate.c`: 프레임 객체를 얻고 `Py_XNewRef`, 호출 하나와 증감 한 쌍), `Py_GetRecursionLimit` + (`Python/ceval.c`: `_PyInterpreterState_GET()`, TLS 읽기), `Py_EnterRecursiveCall` (TLS 읽기와 스택 주소 + 비교, `pycore_ceval.h` `_Py_EnterRecursiveCallTstate`) 를, 나갈 때 `Py_LeaveRecursiveCall` (3.15 에서 빈 함수) + 을 부른다. + - **파이썬 깊이:** 컴파일 프레임이 도는 동안 그 밑의 파이썬 프레임은 바뀌지 않는다(사이에 도는 파이썬 + 코드는 쌓았던 프레임을 다 걷고 돌아온다). 그래서 wrapper 가 한 번 재고 `tp_tls` 에 "구간 깊이" 로 두고 + 나갈 때 되돌리면, 프레임마다의 `PyThreadState_GetFrame` 이 없어진다. 첫 측정이 프레임 객체를 이미 + 만들고 붙잡으므로 새로 생기는 프레임 객체는 없다. 기대 이득: 프로파일의 `PyThreadState_GetFrame` 83 샘플, + 약 5% (추정, #273 전 몫). **에이전트 제안, 확인되지 않음.** `tp_tls`, wrapper, 빠른 진입(#147)을 함께 + 바꾸므로 이 단계에서는 하지 않았다. + - **재귀 한도:** `Py_SetRecursionLimit` 는 3.15t 에서 stop-the-world 로, GIL 빌드에서는 GIL 을 쥐고 바꾼다. + 그래서 다른 스레드는 이 스레드가 safepoint 에 닿기 전에는 바꿀 수 없다. 하지만 이 스레드 안에서는 어느 + `Py_DECREF` 든 `__del__` 을 부를 수 있고, 거기서 한도를 바꿀 수 있다. 바뀐 것을 알 공개 수단이 없으므로 + 프레임마다 읽는 것을 그대로 둔다. + - **C 스택 가드:** tstate 를 받는 공개 변형이 없다. 몇 프레임마다만 부르면 C 스택이 먼저 닿는 경우 + RecursionError 가 나는 프레임이 달라진다. 그대로 둔다. +6. **기각된 것(위 제약).** 타입별 freelist, GC 추적 늦추기·생략, 할당 계수 묶기·임계값 조정. 참고로 3.15t 는 + 이미 스스로 할당 계수를 스레드별로 묶는다(`record_allocation`, `LOCAL_ALLOC_COUNT_THRESHOLD` 512). 이것은 + CPython 자신의 동작이고 컴파일 코드는 그대로 따른다. + +**구현한 것: 참조 옮기기와 빌린 피연산자 (#125, 에이전트 제안, 확인되지 않음).** 생성 C 에서 두 종류의 +1/-1 +쌍을 뺐다. 객체·할당·GC 추적·슬롯 값은 하나도 바뀌지 않는다. + +- **New 의 참조 옮기기.** `Node(make(d-1), make(d-1))` 는 인자 임시값을 `tp_new_fixed` 가 `Py_NewRef` 로 + 슬롯에 복사하고 호출자가 바로 놓았다. 이제 `tp_new_fixed_move` 가 임시값의 참조를 슬롯으로 옮기고 + 호출자는 임시값을 비운다(`_moved_temps`). 실패하면 아무것도 가져가지 않는다: 이미 쓴 슬롯은 반쯤 만든 + 객체를 놓기 전에 NULL 로 되돌린다. 비순수 함수의 느린 경로(`tp_call`)는 그대로 복사·해제한다. +- **빌린 피연산자.** Is 의 두 쪽, FieldGet·IsExact 의 객체, C 간 Call 의 OBJ 인자가 일반 OBJ 지역이면 복사 + (`Py_NewRef` + 해제) 없이 지역 이름(`l_x`)을, None 상수면 `Py_None` 을 바로 쓴다(`_borrow_operand`, + `_operand_release`). 지역의 자기 참조가 그 helper 가 끝날 때까지 객체를 살려 둔다. 이것은 이 프레임의 + 참조이므로 다른 스레드가 무엇을 해도 그렇다. IR 에는 대입식이 없어서 지역은 문장 경계에서만 바뀌고, + 그래서 helper 가 돌 때 읽는 지역은 피연산자를 평가할 때의 지역과 같다. 슬롯 **값**은 여전히 + `tp_field_get` (`PyMember_GetOne`) 으로 새 참조를 잡는다(D5 그대로). +- **관측되지 않는 근거.** 뺀 쌍은 모두 "다른 참조가 그동안 살려 두는 객체의 +1 뒤 -1" 이고, 그 사이에는 이 + 스레드에서 코드가 돌지도 safepoint 에 닿지도 않는다(New 의 근거는 위 #371 과 같다). 같은 스레드는 그 사이를 + 볼 수 없다. 다른 스레드(3.15t)가 그 사이에 `sys.getrefcount` 를 읽을 수 있었다면, 이제 읽는 값은 원래 + 실행에서도 그 창 밖에서 읽었을 값이다. 그래서 새 실행의 관측은 원래 실행에서 가능한 관측의 부분집합이다. + 호출 경계의 refcount, `gc.is_tracked`, `gc.get_objects`, `tp_traverse` 가 보는 슬롯, weakref, `__del__` + 순서, `id()` 는 변하지 않는다. 인터프리터 쪽 근거: 3.15.0rc1t 의 `dis` 로 `check` 를 보면 이 피연산자들이 + 모두 `LOAD_FAST_BORROW` 다(2026-11-13 관측). 인터프리터도 여기서 증감하지 않는다. +- **양.** binary-trees 의 `check` 한 번에 8쌍(FieldGet 의 `node` 2, Is 의 지역 2와 None 2, 재귀 인자 2), + `make` 의 노드 하나에 2쌍(자식 두 개, 잎에서는 None 두 개)이 빠진다. 기대 이득: 프로파일의 "compiled code" + 26% 가 이 증감을 포함한다. 3~8% 로 어림한다(추정, 측정 안 함). 측정은 CI 또는 조용한 창에서 리더가 한다. +- **남은 다음 단계(에이전트 제안, 확인되지 않음):** 대입되지 않는 OBJ 매개변수의 입양(`l_x = Py_NewRef(p_x)`, + 규칙 1)도 뺄 수 있다. 호출자가 호출 내내 인자를 살려 둔다(vectorcall 규약, C 간 호출은 호출자의 지역이나 + 임시값). `check` 한 번에 1쌍. 규칙 1 과 exit 블록을 바꾸므로 이 단계에서는 하지 않았다. +- **시험:** `python-multiplatform-ksp/src/test/python/test_alloc_preserving.py`. 생성 C 구조 시험 셋(구현 전 + 빨강), 인터프리터와 비교하는 관측 시험(`gc.is_tracked`, `gc.get_objects`, 살아 있는 동안 `id` 유일성, + weakref 콜백 순서, `__del__` 순서와 횟수, 뒤에 `__del__` 을 얻는 클래스, `__del__` 있는 서브클래스, 호출 경계의 + `sys.getrefcount`, 많은 호출 뒤 refcount, 4 스레드 동시 실행 뒤 refcount; 구현 전에도 초록, 보존을 지키는 + 시험), 변이 시험 셋(옮긴 임시값을 또 놓기, 옮기지 않고 복사하기 `-DTP_MOVED_REF(v)=Py_NewRef(v)`, 빌린 + 지역을 놓기: 각각 refcount 나 `__del__` 시점으로 잡혀야 한다). + ### 4.3.5 소유권 추론, 메모리, 인터프리터 최소 상주 (설계, 결정 2026-10-05) 사용자 질문 (2026-10-05): "GC의 동작을 smart pointer로 전환하는 컴파일러를 개발하는게 가능할까?", 그리고 diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 10baaa1a2..2d5b0af31 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -81,12 +81,23 @@ 2. every OBJ-typed expression evaluates into a fresh temporary that owns a new reference; temporaries are NULL at every statement boundary; 3. a temporary is consumed exactly once: moved into a local (the local's old reference is - released), moved into `*tp_out` by Return, or released (`tp_release`) right after the helper - that borrowed it returns; + released), moved into `*tp_out` by Return, moved into the slots of a fresh object by + `tp_new_fixed_move` (then emptied, `_moved_temps`), or released (`tp_release`) right after the + helper that borrowed it returns; + 3a. an operand that a helper only borrows for its own duration, the two sides of Is, the object of + a FieldGet or IsExact and the OBJ arguments of a C to C Call, names a plain OBJ local itself + (`l_x`, or `Py_None` for the None constant) instead of a copy: no Py_NewRef, no release + (`_borrow_operand`, `_operand_release`). The local's own reference keeps the object alive for + the helper's whole duration, whatever other threads do (it is this frame's reference), and a + local changes only at a statement boundary (the IR has no assignment expression), so reading + the name when the helper runs is reading it when the operand was evaluated. CPython 3.14+ does + the same for these operands (LOAD_FAST_BORROW). Issue #125; + design 4.3.4 "참조 옮기기와 빌린 피연산자"; 4. every exit of the impl, return, error, deopt, goes through one exit block that releases every OBJ local and temporary. The generated code never decrements a reference except through `tp_release` (API.md); the only - increment is `Py_NewRef` when an OBJ local is read or an OBJ parameter is adopted. + increment is `Py_NewRef` when an OBJ local is read as a value (not as a borrowed operand, 3a) or + an OBJ parameter is adopted. Local arrays (ir.NewArray / CopyArray, `Function` docstring "Local arrays"). A local of an array type is a `tp_i64_array` / `tp_f64_array` struct held by value in the impl, zero-initialised at entry @@ -117,7 +128,8 @@ impure functions (whose impl may never return 1), the impl's copy serves C to C Calls, where a failure is a deopt, so a Call of a function with a `Param.cls` parameter counts as a node that can deopt (`_can_deopt`), exactly like a callee with entry_globals; - 3. FieldGet / FieldSet / New go through `tp_field_get` / `tp_field_set` / `tp_new_fixed` with the + 3. FieldGet / FieldSet / New go through `tp_field_get` / `tp_field_set` / `tp_new_fixed_move` + (tp_new_fixed's stores, with the argument temporaries' references moved in, #125) with the captured descriptors; CheckExact deopts (return 1) on mismatch and is legal only in a pure function; Is is a plain BOOL. @@ -449,6 +461,20 @@ def _class_current_expr(obj_c: str, k: int, fields_c: str, n: int, descrs_c: str f"{fields_c}, {n}, {descrs_c})") +def _moved_temps(items: list[str]) -> str: + """After `tp_new_fixed_move` succeeded: the object owns the argument temporaries' references, so + the temporaries are emptied instead of released (module docstring, rule 3; #125). The single + mutation point of the move: returning "" releases them anyway and must fail the tests.""" + return " ".join(f"{t} = NULL;" for t in items) + + +def _operand_release(name: str, owned: bool) -> str | None: + """The temporary to release after a helper borrowed operand `name`: an owned temporary, never a + borrowed OBJ local (rule 3a; #125). The single mutation point of the borrow: releasing every + operand drops a reference the local still owns and must fail the tests.""" + return name if owned else None + + def _watch_event_stmt() -> str: """The body of the module's dict-watcher callback (module docstring, rule 5).""" return "return tp_globals_watch_event(event, key, tp_watch_names);" @@ -1521,15 +1547,18 @@ def ev_Call(self, e: ir.Call) -> str | None: int_arg = a.type is Type.INT and p.type is Type.I64 and e.redo if a.type is not p.type and not int_arg: raise CGenError(f"{self.f.name}: Call {e.function} arg {p.name}: {a.type} for {p.type}") - v = self.ev(a) + if a.type is Type.OBJ: + vo = self._borrow_operand(a, "Call argument") # rule 3a + v = vo[0] + owned.append(vo) + else: + v = self.ev(a) values.append(v) if int_arg: args.append(f"{v}.v") int_args.append(v) else: args.append(v) - if a.type is Type.OBJ: - owned.append(v) t = ti = None self.emit("tp_s = 0;") if not self.f.pure and not self.fast: @@ -1570,7 +1599,7 @@ def ev_Call(self, e: ir.Call) -> str | None: # a pure callee using classes returns 1 when a class check fails (module docstring, # rule 7): redo it interpreted; this impure caller must not deopt self._redo_call(callee, e.args, values, t) - self._own_release(*owned) + self._release_operands(*owned) for v in int_args: self.emit(f"tp_int_release(&{v});") self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") @@ -1660,17 +1689,35 @@ def _obj_operand(self, e: ir.Expr, what: str) -> str: raise CGenError(f"{self.f.name}: {what} on {e.type}") return self.ev(e) + def _borrow_operand(self, e: ir.Expr, what: str) -> tuple[str, bool]: + """An OBJ operand the helper only borrows (module docstring, rule 3a): (C name, owned). + A plain OBJ local (not virtual, not a region pointer) names itself and the None constant + names Py_None, both borrowed; anything else is an owned temporary as `_obj_operand`.""" + if e.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: {what} on {e.type}") + if not self.rmode: + if (isinstance(e, ir.Local) and e.name not in self.virtual + and self.types.get(e.name) is Type.OBJ): + return _ident("l", e.name), False + if isinstance(e, ir.Const) and e.value is None: + return "Py_None", False + return self.ev(e), True + + def _release_operands(self, *ops: tuple[str, bool]) -> None: + self._own_release(*[n for n in (_operand_release(name, owned) for name, owned in ops) + if n is not None]) + def ev_Is(self, e: ir.Is) -> str: if self.rmode: a, b = self._r_ptr_name(e.left), self._r_ptr_name(e.right) t = self.tmp("int") self.emit(f"{t} = ({a} {'!=' if e.negate else '=='} {b});") return t - a = self._obj_operand(e.left, "Is") - b = self._obj_operand(e.right, "Is") + ao = self._borrow_operand(e.left, "Is") + bo = self._borrow_operand(e.right, "Is") t = self.tmp("int") - self.emit(f"{t} = ({a} {'!=' if e.negate else '=='} {b});") - self._own_release(a, b) + self.emit(f"{t} = ({ao[0]} {'!=' if e.negate else '=='} {bo[0]});") + self._release_operands(ao, bo) return t def _current(self, obj_c: str, cls: str) -> str: @@ -1697,7 +1744,8 @@ def ev_IsExact(self, e: ir.IsExact) -> str: self.emit(f"{t} = ({p} != NULL);") return t k = self._class(e.cls) - o = self._obj_operand(e.obj, "IsExact") + oo = self._borrow_operand(e.obj, "IsExact") + o = oo[0] t = self.tmp("int") self.emit(f"tp_s = {self._global_ok(e.cls)};") self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") @@ -1712,7 +1760,7 @@ def ev_IsExact(self, e: ir.IsExact) -> str: self._own_release(g) self.depth -= 1 self.emit("}") - self._own_release(o) + self._release_operands(oo) return t def ev_CheckExact(self, e: ir.CheckExact) -> str: @@ -1744,7 +1792,8 @@ def ev_FieldGet(self, e: ir.FieldGet) -> str: raise CGenError(f"{self.f.name}: FieldGet {e.cls}.{e.field} on virtual {e.obj.name}") return self._own_copy(_ident("l", _vfield(e.obj.name, cd.fields.index(e.field)))) slot = self.mg.slot(self.f.name, e.cls, e.field) - o = self._obj_operand(e.obj, "FieldGet") + oo = self._borrow_operand(e.obj, "FieldGet") + o = oo[0] t = self.tmp("PyObject *") self.emit(f"tp_s = {self._current(o, e.cls)};") self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") @@ -1757,7 +1806,7 @@ def ev_FieldGet(self, e: ir.FieldGet) -> str: self.emit(f"{t} = tp_getattr({o}, tp_st->names[{self.mg.name_index(e.field)}]);") self.depth -= 1 self.emit("}") - self._own_release(o) + self._release_operands(oo) self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") return t @@ -1811,8 +1860,10 @@ def ev_New(self, e: ir.New) -> str: self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") self.emit(av) self.emit(" if (tp_s) {") - self.emit(f" {t} = tp_new_fixed((PyTypeObject *)tp_st->classes[{k}], " + self.emit(f" {t} = tp_new_fixed_move((PyTypeObject *)tp_st->classes[{k}], " f"{f'&tp_st->slots[{base}]' if items else 'NULL'}, tp_av, {len(items)});") + if items: # the object owns them now (rule 3) + self.emit(f" if ({t} != NULL) {{ {_moved_temps(items)} }}") self.emit(" } else {") self.depth += 1 if self.f.pure: @@ -1824,7 +1875,7 @@ def ev_New(self, e: ir.New) -> str: self.emit(f" {t} = tp_call({gk} ? tp_st->classes[{k}] : {g}, tp_av, {len(items)}, NULL);") self.depth -= 1 self.emit(" } }") - self._own_release(*items) # the slots took their own references + self._own_release(*items) # emptied when moved into the slots; released on the slow path if g is not None: self._own_release(g) self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 4deb3819b..0ca623413 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -400,6 +400,10 @@ Slot access and the slow paths: PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); /* cls->tp_alloc(cls, 0) then a release store of a new reference into each slot (the fresh object is unreachable until it returns, #125); NULL + MemoryError */ + PyObject *tp_new_fixed_move(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); + /* the same stores with values[i] moved in, not copied: on success the object owns them + and the caller empties its temporaries; on failure nothing was taken (#125). + This is what the generator emits for New */ int tp_is_exact(PyObject *obj, PyTypeObject *cls); /* Py_IS_TYPE */ FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 37f4b8a2f..76d75cd4a 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2843,6 +2843,46 @@ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slot return obj; } +/* tp_new_fixed with the caller's references moved into the slots (#125, design 4.3.4 "참조 옮기기"): + * on success the object owns values[i] (the caller drops its pointers without releasing them); on + * failure nothing was taken and the caller still owns every values[i]. + * + * Same stores, same object, same allocation as tp_new_fixed; what is gone is a Py_NewRef of each + * value here and the caller's Py_CLEAR of its temporary right after, a +1/-1 pair on the same object + * with nothing in between that can run code or reach a safepoint. Every reference count another + * thread can read during the call is one it could read with the pair (outside the pair's window), so + * no observation tells the two apart; the count after the call is the same. On failure the slots + * already written are emptied again before the half-built object is released, so its dealloc drops + * no reference the caller still owns. + * TP_MOVED_REF is the single mutation point: test_alloc_preserving.py builds with it defined as + * Py_NewRef (a copy instead of a move), which leaks one reference per slot and must fail. */ +#ifndef TP_MOVED_REF +#define TP_MOVED_REF(v) (v) +#endif +static inline PyObject *tp_new_fixed_move(PyTypeObject *cls, PyMemberDef *const *slots, + PyObject *const *values, Py_ssize_t n) +{ + Py_ssize_t i, j; + PyObject *obj = cls->tp_alloc(cls, 0); + + if (obj == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + if (values[i] == NULL) { + for (j = 0; j < i; j++) { /* give the moved references back to the caller */ + *(PyObject **)((char *)obj + slots[j]->offset) = NULL; + } + PyErr_SetString(PyExc_AttributeError, slots[i]->name); + Py_DECREF(obj); + return NULL; + } + __atomic_store_n((PyObject **)((char *)obj + slots[i]->offset), TP_MOVED_REF(values[i]), + __ATOMIC_RELEASE); + } + return obj; +} + /* type(obj) is cls */ static inline int tp_is_exact(PyObject *obj, PyTypeObject *cls) { diff --git a/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py new file mode 100644 index 000000000..6b28e5b1c --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py @@ -0,0 +1,476 @@ +"""TypedPython object cost without a change in what Python can observe (#125, design 4.3.4 +"참조 옮기기와 빌린 피연산자", cgen module docstring rules 3 and 3a). + +Two changes of the generated C, both removing a +1/-1 reference-count pair on an object that another +reference keeps alive, with nothing in between that can run code or reach a safepoint: + - a New moves its argument temporaries' references into the fresh object's slots + (`tp_new_fixed_move`) instead of copying them and releasing the temporaries; + - an operand a helper only borrows (the two sides of Is, the object of FieldGet / IsExact, an OBJ + argument of a C to C Call) names the OBJ local itself instead of a Py_NewRef copy. + +The behaviour tests compare every observation the leader's constraint (2026-11-13) lists, gc +tracking, gc.get_objects, weakrefs and their callbacks, __del__ order, sys.getrefcount, id +uniqueness, subclasses and a class changed after import, with the interpreted functions the +extension keeps. They pass before and after the change: they guard the preserved behaviour, they do +not measure the change. The mutation tests show that the guarded properties fail when the move or the +borrow is done wrongly (a copy that leaks, a release that frees too early). + +Red record (expected; agents do not build here, AGENTS.md 10, so CI runs this file): + - before the change: test_gen_new_moves_*, test_gen_borrowed_operands_*, test_header_move_* + fail (no tp_new_fixed_move, Py_NewRef copies of locals), and the three mutation tests fail with + AttributeError (no cgen._moved_temps / cgen._operand_release) or a compile error (no + TP_MOVED_REF); every behaviour test passes; + - after the change: everything passes. +""" +from __future__ import annotations + +import gc +import subprocess +import sys +import textwrap +import threading +import weakref + +import pytest + +from test_cgen import PACKAGE, OUT_ROOT, BOOL, OBJ, L, fn, runtime_dir, need_runtime +from test_cgen_classes import f_make, f_check +from typedpython.ir import ClassDecl, FieldGet, Is, Module, New, Param, Return, Const + +SOURCE = textwrap.dedent('''\ + """typedpython allocation test module, the interpreted reference.""" + LOG = [] + + class Node: + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + class WNode: # weak-referenceable + __slots__ = ("left", "right", "__weakref__") + + def __init__(self, left, right): + self.left = left + self.right = right + + class DNode: # a finalizer from the start + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + def __del__(self): + LOG.append(id(self)) + + class LNode: # gains __del__ after import (a test sets it) + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + def make(depth): + if depth == 0: + return None + return Node(make(depth - 1), make(depth - 1)) + + def check(node): + if node is None: + return 0 + return 1 + check(node.left) + check(node.right) + + def make_w(depth): + if depth == 0: + return None + return WNode(make_w(depth - 1), make_w(depth - 1)) + + def make_d(depth): + if depth == 0: + return None + return DNode(make_d(depth - 1), make_d(depth - 1)) + + def make_l(depth): + if depth == 0: + return None + return LNode(make_l(depth - 1), make_l(depth - 1)) + + def pair(x): + return Node(x, x) + + def first(p): + return p.left + + def is_none(x): + return x is None + ''') + +CLASSES = (ClassDecl("Node", ("left", "right"), True), ClassDecl("WNode", ("left", "right"), True), + ClassDecl("DNode", ("left", "right"), True, has_del=True), + ClassDecl("LNode", ("left", "right"), True)) + + +def f_pair(): + x = L("x", OBJ) + return fn("pair", [Param("x", OBJ)], OBJ, {}, [Return(New(OBJ, "Node", (x, x)))], + pure=True, may_deopt=True) + + +def f_first(): + return fn("first", [Param("p", OBJ, cls="Node")], OBJ, {}, + [Return(FieldGet(OBJ, L("p", OBJ), "Node", "left"))], pure=True, may_deopt=True) + + +def f_is_none(): + return fn("is_none", [Param("x", OBJ)], BOOL, {}, + [Return(Is(BOOL, L("x", OBJ), Const(OBJ, None)))], pure=True, may_deopt=False) + + +def functions(): + return (f_make(), f_check(), f_make("make_w", "WNode"), f_make("make_d", "DNode"), + f_make("make_l", "LNode"), f_pair(), f_first(), f_is_none()) + + +def generate(name): + import shutil + from typedpython import cgen + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return cgen.generate(Module(name=name, functions=functions(), classes=CLASSES), p) + + +def build(name, flags=None): + from typedpython import cbuild + need_runtime() + src = generate(name) + kwargs = {} if flags is None else {"flags": flags} + so = cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir(), **kwargs) + return cbuild.load(name, so), so + + +@pytest.fixture(scope="module") +def m(): + return build("tp_alloc_pres")[0] + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def nodes(t): + """Every node of a tree, pre-order.""" + out, todo = [], [t] + while todo: + n = todo.pop() + if n is None: + continue + out.append(n) + todo.append(n.right) + todo.append(n.left) + return out + + +def impl_body(text, name): + """The definition (not the forward declaration) of the counted impl of `name`.""" + sig = f"static int tp_impl_{name}(PyObject *tp_module, PyThreadState" + start = text.index(sig) + while text[start:text.index("\n", start)].endswith(";"): # skip the declaration + start = text.index(sig, start + 1) + return text[start:text.index("\n}\n", start)] + + +# --- the generated C (red before the change) -------------------------------------------------------- + +def test_gen_new_moves_its_argument_temporaries(): + text = generate("tp_alloc_gen_new") + assert "tp_new_fixed(" not in text, "a New still copies its arguments (tp_new_fixed)" + lines = text.splitlines() + sites = [i for i, ln in enumerate(lines) if "= tp_new_fixed_move(" in ln] + assert sites, text + for i in sites: + t = lines[i].split("=")[0].strip() + nxt = lines[i + 1].strip() + assert nxt.startswith(f"if ({t} != NULL) {{") and "= NULL;" in nxt, (lines[i], nxt) + + +def test_gen_borrowed_operands_take_no_copy(): + text = generate("tp_alloc_gen_borrow") + check = impl_body(text, "check") + # adoption of the parameter stays (rule 1); no copy of the local + assert "l_node = Py_NewRef(p_node);" in check and "Py_NewRef(l_node)" not in check, check + assert "(l_node == Py_None)" in check, check + assert "tp_field_get(l_node," in check, check + assert "tp_impl_check(tp_module, tp_ts, tp_t, l_l," in check, check + assert "tp_impl_check(tp_module, tp_ts, tp_t, l_r," in check, check + assert "Py_NewRef(Py_None)" not in check, check + assert "tp_release(&l_" not in check.split("tp_exit:")[0], "a borrowed local is released" + first = impl_body(text, "first") + assert "tp_field_get(l_p," in first and "Py_NewRef(l_p)" not in first, first + + +def test_header_move_helper_moves_and_gives_back_on_failure(): + from typedpython import cbuild + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + start = h.index("static inline PyObject *tp_new_fixed_move(") + body = h[start:h.index("\n}\n", start)] + assert "Py_NewRef(" not in body, body # the caller's reference moves in + assert "TP_MOVED_REF(values[i])" in body, body # the single mutation point + assert "__atomic_store_n(" in body and "__ATOMIC_RELEASE" in body, body + assert "PyMember_SetOne" not in body and "CRITICAL_SECTION" not in body, body + # on a failure after some stores, those slots are emptied before the object is released + fail = body[body.index("values[i] == NULL"):] + assert fail.index("= NULL;") < fail.index("Py_DECREF(obj)"), body + + +# --- observable behaviour, compiled against interpreted (green before and after) ------------------- + +def test_results_match_the_interpreter(m): + for d in (0, 1, 5): + assert m.check(m.make(d)) == interp(m, "check")(interp(m, "make")(d)) == 2 ** d - 1 + x = object() + p, q = m.pair(x), interp(m, "pair")(x) + assert type(p) is type(q) is m.Node and p.left is p.right is x and q.left is x + assert m.first(p) is x and m.is_none(None) is True and m.is_none(p) is False + + +def test_compiled_objects_are_gc_tracked_like_interpreted_ones(m): + t, u = m.make(4), interp(m, "make")(4) + assert [gc.is_tracked(n) for n in nodes(t)] == [gc.is_tracked(n) for n in nodes(u)] + assert all(gc.is_tracked(n) for n in nodes(t)) + assert gc.is_tracked(m.pair(None)) and gc.is_tracked(interp(m, "pair")(None)) + + +def test_compiled_objects_are_in_gc_get_objects(m): + t = m.make(5) + live = {id(o) for o in gc.get_objects() if type(o) is m.Node} + assert all(id(n) in live for n in nodes(t)) + assert len(nodes(t)) == 31 + + +def test_ids_are_unique_while_alive(m): + t = m.make(7) + keep = [m.pair(None) for _ in range(100)] + ids = [id(n) for n in nodes(t)] + [id(p) for p in keep] + assert len(ids) == len(set(ids)) == 127 + 100 + + +def test_weakref_callbacks_run_in_the_interpreters_order(m): + def order(make): + t = make(4) + index = {id(n): k for k, n in enumerate(nodes(t))} + log = [] + refs = [weakref.ref(n, lambda r, k=index[id(n)]: log.append(k)) for n in nodes(t)] + assert all(r() is not None for r in refs) + del t + assert all(r() is None for r in refs) + return log + got, want = order(m.make_w), order(interp(m, "make_w")) + assert got == want and sorted(got) == list(range(15)), (got, want) + + +def test_del_runs_once_per_object_in_the_interpreters_order(m): + def order(make): + m.LOG.clear() + t = make(4) + index = {id(n): k for k, n in enumerate(nodes(t))} + assert m.LOG == [] + del t + return [index[i] for i in m.LOG] + got, want = order(m.make_d), order(interp(m, "make_d")) + assert got == want and sorted(got) == list(range(15)), (got, want) + + +def test_a_class_that_gains_del_later_finalizes_compiled_objects(m): + t = m.make_l(3) + m.LNode.__del__ = lambda self: m.LOG.append(id(self)) + try: + m.LOG.clear() + ids = {id(n) for n in nodes(t)} + del t + assert set(m.LOG) == ids and len(m.LOG) == 7 + m.LOG.clear() + u = m.make_l(3) # built after the change (the class's tag moved) + ids = {id(n) for n in nodes(u)} + del u + assert set(m.LOG) == ids and len(m.LOG) == 7 + finally: + del m.LNode.__del__ + m.LOG.clear() + + +def test_a_subclass_with_del_inside_a_compiled_pair_dies_when_the_pair_dies(m): + log = [] + + class Sub(m.Node): + __slots__ = () + + def __del__(self): + log.append("sub") + + s = Sub(None, None) + p = m.pair(s) + del s + assert log == [] # the pair's two slots keep it alive + assert m.check(Sub(None, None)) == interp(m, "check")(Sub(None, None)) == 1 + log.clear() + del p + assert log == ["sub"] + + +def test_reference_counts_match_the_interpreter_at_every_call_boundary(m): + x = object() + base = sys.getrefcount(x) + p = m.pair(x) + compiled = sys.getrefcount(x) + q = interp(m, "pair")(x) + assert compiled == base + 2 and sys.getrefcount(x) == base + 4 + del p, q + n, h = m.Node(None, None), m.Node(x, None) + b = sys.getrefcount(n), sys.getrefcount(h), sys.getrefcount(x) + for _ in range(50): + m.check(n), m.first(h), m.is_none(n), m.check(m.Node(n, n)) + assert (sys.getrefcount(n), sys.getrefcount(h), sys.getrefcount(x)) == b + + +def test_reference_counts_are_unchanged_over_many_calls(m): + shared = object() + holder = m.Node(shared, None) + keep = m.Node(None, None) + + def churn(k): + for _ in range(k): + m.check(m.make(4)) + m.pair(shared) + m.first(holder) + m.is_none(keep) + m.check(m.Node(keep, None)) + + churn(20) + base = sys.getrefcount(shared), sys.getrefcount(holder), sys.getrefcount(keep) + churn(3000) + assert (sys.getrefcount(shared), sys.getrefcount(holder), sys.getrefcount(keep)) == base + + +def test_concurrent_threads_keep_counts_exact(m): + """3.15t: other threads move a shared object's count through ob_ref_shared, this one through + ob_ref_local; a lost or extra increment shows in sys.getrefcount once the threads are done.""" + shared = object() + keep = m.Node(None, None) + base = sys.getrefcount(shared), sys.getrefcount(keep) + errors = [] + + def work(): + try: + for _ in range(3000): + p = m.pair(shared) + assert m.first(p) is shared + assert m.check(m.Node(keep, keep)) == 3 + assert m.check(m.make(3)) == 7 + assert m.is_none(keep) is False + except Exception as e: # noqa: BLE001 (reported below) + errors.append(repr(e)) + + threads = [threading.Thread(target=work) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + assert (sys.getrefcount(shared), sys.getrefcount(keep)) == base + + +# --- mutations: a wrong move or borrow must fail ---------------------------------------------------- + +MUTANT_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + log = [] + class T: + def __del__(self): + log.append("T") + bad = 0 + x = T() + base = sys.getrefcount(x) + p = m.pair(x) + if sys.getrefcount(x) != base + 2: + bad |= 1 + for _ in range(200): + m.pair(x) + m.check(m.make(4)) + if sys.getrefcount(x) != base + 2: + bad |= 2 + del x + if log: + bad |= 4 # freed while the pair still holds it + del p + if log != ["T"]: + bad |= 8 + n = m.Node(None, None) + b = sys.getrefcount(n) + for _ in range(100): + m.check(n) + m.first(n) + m.is_none(n) + if sys.getrefcount(n) != b: + bad |= 16 + print("RESULT", bad, log) + sys.exit(bad) + ''') + + +def run_child(name, so): + script = MUTANT_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=120) + + +def test_mutation_unmutated_build_passes_the_script(): + _, so = build("tp_alloc_ok") + run = run_child("tp_alloc_ok", so) + assert run.returncode == 0, run.stdout + run.stderr + + +def _build_mutant(name, monkeypatch, attr, value): + from typedpython import cgen, cbuild + need_runtime() + monkeypatch.setattr(cgen, attr, value) + src = generate(name) + monkeypatch.undo() + return cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir()) + + +def test_mutation_releasing_moved_temporaries_is_caught(monkeypatch): + """The temporaries a New moved into its object are released anyway: one reference too few per + slot, the argument dies while the pair still points at it.""" + so = _build_mutant("tp_alloc_mut_move", monkeypatch, "_moved_temps", lambda items: "") + run = run_child("tp_alloc_mut_move", so) + assert run.returncode != 0, "the over-release mutant passed:\n" + run.stdout + run.stderr + + +def test_mutation_copying_instead_of_moving_is_caught(): + """tp_new_fixed_move takes a new reference as tp_new_fixed did, while the caller still drops its + temporaries without releasing them: one leaked reference per slot.""" + from typedpython import cbuild + flags = list(cbuild.DEFAULT_FLAGS) + ["-DTP_MOVED_REF(v)=Py_NewRef(v)"] + need_runtime() + src = generate("tp_alloc_mut_copy") + so = cbuild.build(src, "tp_alloc_mut_copy", OUT_ROOT / "tp_alloc_mut_copy", + runtime_dir=runtime_dir(), flags=flags) + run = run_child("tp_alloc_mut_copy", so) + assert run.returncode != 0, "the leaking mutant passed:\n" + run.stdout + run.stderr + + +def test_mutation_releasing_a_borrowed_local_is_caught(monkeypatch): + """A borrowed OBJ local is released after the helper as if it were a temporary: the local's own + reference is dropped (and the local emptied) while the function still uses it.""" + so = _build_mutant("tp_alloc_mut_borrow", monkeypatch, "_operand_release", + lambda name, owned: name if (owned or name.startswith("l_")) else None) + run = run_child("tp_alloc_mut_borrow", so) + assert run.returncode != 0, "the borrowed-release mutant passed:\n" + run.stdout + run.stderr From 6aab55f4d1b441458ea29bc1b0dbf02b21e8d47f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:42:19 +0900 Subject: [PATCH 50/61] TypedPython: the copy-instead-of-move mutant is an object-like switch (#125) On Windows the mutant built with -DTP_MOVED_REF(v)=Py_NewRef(v) was not caught (run 43362638660): cl.exe takes no function-like macro on its command line, so it most likely built the unmutated move. -DTP_MUTANT_COPY_MOVED selects the copy in the header for every compiler. --- docs/design/typedpython.md | 2 +- .../src/main/python/typedpython/runtime/tp_runtime.h | 9 +++++++-- .../src/test/python/test_alloc_preserving.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 7540b1685..ad3784516 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -517,7 +517,7 @@ CPython 출처는 v3.15.0rc1 태그의 파일과 함수 이름이다. 빨강), 인터프리터와 비교하는 관측 시험(`gc.is_tracked`, `gc.get_objects`, 살아 있는 동안 `id` 유일성, weakref 콜백 순서, `__del__` 순서와 횟수, 뒤에 `__del__` 을 얻는 클래스, `__del__` 있는 서브클래스, 호출 경계의 `sys.getrefcount`, 많은 호출 뒤 refcount, 4 스레드 동시 실행 뒤 refcount; 구현 전에도 초록, 보존을 지키는 - 시험), 변이 시험 셋(옮긴 임시값을 또 놓기, 옮기지 않고 복사하기 `-DTP_MOVED_REF(v)=Py_NewRef(v)`, 빌린 + 시험), 변이 시험 셋(옮긴 임시값을 또 놓기, 옮기지 않고 복사하기 `-DTP_MUTANT_COPY_MOVED`, 빌린 지역을 놓기: 각각 refcount 나 `__del__` 시점으로 잡혀야 한다). ### 4.3.5 소유권 추론, 메모리, 인터프리터 최소 상주 (설계, 결정 2026-10-05) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 76d75cd4a..b7340f71a 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2854,11 +2854,16 @@ static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slot * no observation tells the two apart; the count after the call is the same. On failure the slots * already written are emptied again before the half-built object is released, so its dealloc drops * no reference the caller still owns. - * TP_MOVED_REF is the single mutation point: test_alloc_preserving.py builds with it defined as - * Py_NewRef (a copy instead of a move), which leaks one reference per slot and must fail. */ + * TP_MOVED_REF is the single mutation point: test_alloc_preserving.py builds with -DTP_MUTANT_COPY_MOVED, + * which defines it as Py_NewRef (a copy instead of a move), leaks one reference per slot and must fail. */ #ifndef TP_MOVED_REF +#ifdef TP_MUTANT_COPY_MOVED /* the mutation: a copy instead of a move (an object-like switch, as cl.exe + * cannot define a function-like macro on its command line) */ +#define TP_MOVED_REF(v) Py_NewRef(v) +#else #define TP_MOVED_REF(v) (v) #endif +#endif static inline PyObject *tp_new_fixed_move(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n) { diff --git a/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py index 6b28e5b1c..2b43b4cdd 100644 --- a/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py +++ b/python-multiplatform-ksp/src/test/python/test_alloc_preserving.py @@ -458,7 +458,7 @@ def test_mutation_copying_instead_of_moving_is_caught(): """tp_new_fixed_move takes a new reference as tp_new_fixed did, while the caller still drops its temporaries without releasing them: one leaked reference per slot.""" from typedpython import cbuild - flags = list(cbuild.DEFAULT_FLAGS) + ["-DTP_MOVED_REF(v)=Py_NewRef(v)"] + flags = list(cbuild.DEFAULT_FLAGS) + ["-DTP_MUTANT_COPY_MOVED"] # object-like: cl.exe too need_runtime() src = generate("tp_alloc_mut_copy") so = cbuild.build(src, "tp_alloc_mut_copy", OUT_ROOT / "tp_alloc_mut_copy", From d524f45b4bff27af4cb67bb2c88a031190730b59 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:50:46 +0900 Subject: [PATCH 51/61] TypedPython test: the array-copy handler test stays under one poll interval (#406) Each poll of a function holding an array writes the list back and copies it in again (#152, #335). With 100_000 elements total() made 480_000 full re-copies and timed out at 120 s on slow macOS runners. The list now has 4_000 elements (no inner-loop poll) and 500_000 rounds keep the 2e9 additions; the test is renamed for what it checks, and the comments in tp_runtime.h and the design note that still said such functions do not poll are corrected. --- docs/design/typedpython.md | 3 ++- .../python/typedpython/runtime/tp_runtime.h | 18 +++++++------- .../src/test/python/test_eval_breaker.py | 24 ++++++++++++------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 686ebc722..883f9fc90 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -400,7 +400,8 @@ PyPy·GraalVM 과 같은 방식이다. N-11 클래스의 객체를 PyObject 대 eval breaker 처리를 그대로 돌린다. 배열 copy-in·전역 스냅샷을 쥔 함수가 실행 중이면 미룬다. 그 함수 안에서의 GIL 넘기기는 2단계(#154)이고, 방식은 결정 대기(#152). **비용 (2026-10-17, 조용한 창, develop 대비 교대 11쌍):** fannkuch +8.2% (중앙값 1.082, 범위 1.043~1.191), - nbody 0.995 (배열 스냅샷 함수는 폴링하지 않음). 루프 폴링을 뺀 실험판은 fannkuch 1.000 이므로 남은 비용은 + nbody 0.995 (배열 스냅샷 함수는 폴링하지 않음; 이 측정 당시. #335(2026-10-25) 이후 스냅샷 함수도 폴링하며 폴링마다 + 리스트를 되쓰고 다시 복사한다, SPEC N-8, #406). 루프 폴링을 뺀 실험판은 fannkuch 1.000 이므로 남은 비용은 전부 짧은 안쪽 루프의 매 반복 카운트다운이다. CPython 도 모든 back-edge 에서 eval breaker 를 검사하므로 같은 종류의 비용이고, 시그널·스레드 굶김과 3.15t 의 GC stop-the-world 차단을 고치는 대가로 받아들였다(리더 결정). range 루프의 묶음 폴링은 후속 이슈다. 원자료: `benchmarks/typedpython/results/2026-10-17-eval-breaker-*.json`. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 37f4b8a2f..fa995ccc5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1005,10 +1005,10 @@ static inline int tp_unbox_bool(PyObject *o, int *out) * borrowed references (PyList_GET_ITEM) and rely on that lock: no other thread can change the list * between the copy-in and the write-back, so the whole call is one atomic step as far as lists are * concerned (CPython can produce that interleaving: every list operation takes the same lock). - * Liveness note (issue #152): a function holding an array does not poll the eval breaker, so a long - * compiled array loop delays other threads' stop-the-world requests (the free-threaded GC) until it - * returns; results are unaffected. Polls inside such functions need the re-validation decided in - * #152 (after a poll the critical section is re-taken and the list may have changed). + * Polls (issue #152, #335, SPEC N-8): a function holding an array polls once per TP_POLL_INTERVAL + * at the top of a loop iteration: it writes the array back with exit, ends the critical section, + * polls, and enters again (re-taking the section and re-checking the list), so each poll costs a + * full write-back and copy-in of the list (#406). * exit: writes the dirty slots back as new objects (PyList_SetItem steals the new reference and, on * failure, drops it), always frees, and zeroes the struct. On a write-back failure it stops, * frees, and returns -1 with that error set (MemoryError, or IndexError if the list shrank). @@ -3052,11 +3052,11 @@ static inline void tp_trace_fast_hit(void) {} * (forced GIL switching to a waiting thread, KeyboardInterrupt, gc). Public API only. * * Signal handlers, pending calls and other threads are user code that may change lists and - * globals. A function holding an array copy-in or an entry-globals snapshot runs no user code by - * contract (ir.ArrayParam, ir.Function.entry_globals), so such a function does not poll, and while - * one is active (`tp_snapshot_depth > 0`, counted at its entry and exit) the slow path does nothing: - * the event stays pending and is handled at the next poll or bytecode after it returns, which is the - * CPython execution in which it arrived late. + * globals. A function holding an array copy-in or an entry-globals snapshot runs no user code while + * it holds them (ir.ArrayParam, ir.Function.entry_globals): while one is active + * (`tp_snapshot_depth > 0`, counted at its entry and exit) the slow path does nothing and the event + * stays pending. Its own polls (#152, #335) first write the arrays back and lower the depth, so the + * handler runs there with the lists up to date, and the function copies them in again after. * ------------------------------------------------------------------------------------------ */ #ifndef TP_POLL_INTERVAL #define TP_POLL_INTERVAL 4096 diff --git a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py index faea4a5ae..d02182c6d 100644 --- a/python-multiplatform-ksp/src/test/python/test_eval_breaker.py +++ b/python-multiplatform-ksp/src/test/python/test_eval_breaker.py @@ -2,9 +2,10 @@ signals reach Python (KeyboardInterrupt), and other threads get the GIL. A Python signal handler, a pending call and another thread are user code that can change lists and -globals. A function that holds an array copy-in (ArrayParam) or an entry-globals snapshot runs no -user code by contract, so while such a function is active the poll is deferred: the result is then -the CPython execution in which the signal arrived after the call. +globals. A function that holds an array copy-in (ArrayParam) or an entry-globals snapshot polls +too (#152, #335, SPEC N-8): at a poll it writes its lists back, runs the handler, copies them in +again and re-checks its entry guards, so the result includes what the handler wrote (or, after a +poll it cannot continue from, it leaves for the interpreted continuation). Each scenario runs in a child interpreter, so a broken build cannot take pytest down and signal handlers stay out of the test process. Timing thresholds are generous; they separate "polled" @@ -136,12 +137,17 @@ def worker(): @posix_signals -def test_a_handler_is_deferred_while_an_array_copy_is_live(extension): +def test_a_handler_during_an_array_copy_is_seen_by_the_result(extension): # total() reads its list through a copy-in. A handler that writes the list mid-call would be - # invisible to the copy, a result no CPython execution can give. So the handler must run after - # the call (the execution where the signal arrived late), or the result must include its write. + # invisible to a stale copy, a result no CPython execution can give. So either the handler runs + # after the call (the execution where the signal arrived late) or the result includes its write + # (a poll wrote the list back, ran the handler and copied it in again, #152/#335). + # Size (#406): each poll inside the call re-copies the whole list, so the list stays under + # TP_POLL_INTERVAL (4096) and the inner loop never polls; with 100_000 elements a call made + # 480_000 full re-copies and timed out at 120 s on slow macOS runners. The outer loop still polls + # (once per 4096 rounds); the inner-loop poll and the OSR exit are covered by test_osr_exit.py. r = run_child(extension, """ - xs = [1.0] * 100_000 + xs = [1.0] * 4_000 seen = {} def handler(signum, frame): seen["t"] = time.perf_counter() @@ -150,10 +156,10 @@ def handler(signum, frame): import os threading.Timer(0.05, lambda: os.kill(os.getpid(), signal.SIGUSR1)).start() t0 = time.perf_counter() - s = m.total(xs, 20_000) + s = m.total(xs, 500_000) # 2e9 additions, as before t1 = time.perf_counter() time.sleep(0.01) # let a deferred handler run - print(json.dumps({"s": s, "plain": 100_000 * 20_000 * 1.0, "t0": t0, "t1": t1, + print(json.dumps({"s": s, "plain": 4_000 * 500_000 * 1.0, "t0": t0, "t1": t1, "handler": seen.get("t")})) """) assert r["handler"] is not None, "the handler never ran" From 9ea48a20dd2d934c3321efe7191551ec6a5b1a82 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:19:32 +0900 Subject: [PATCH 52/61] TypedPython tests: hold the two race mutants' windows open in the mutant build (#399) The dict-critical-section mutant now waits in the watcher, after the epoch bump, until a reader has remembered the old binding under the new epoch; the borrowed-field-read mutant waits until the slot was replaced before taking its late reference. Both waits are bounded and exist only in the mutant headers. The shared field body checks (n, [n]) results, which the real build must still pass with no error. --- .../src/test/python/test_free_threaded.py | 76 +++++++++++++++++-- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/python-multiplatform-ksp/src/test/python/test_free_threaded.py b/python-multiplatform-ksp/src/test/python/test_free_threaded.py index cdbd52cce..88db26a2b 100644 --- a/python-multiplatform-ksp/src/test/python/test_free_threaded.py +++ b/python-multiplatform-ksp/src/test/python/test_free_threaded.py @@ -939,8 +939,45 @@ def reader(): assert detected, "the mutant (no critical section around copy-in) was not caught" -def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool) -> Path: - """A copy of tp_runtime.h with one #159 protection removed (the mutation of that protection).""" +# Mutant-build-only scheduling helpers (#399). A mutant of the real header may contain these; the real +# header never does. TP_MUT_YIELD gives up the CPU (POSIX sched_yield, Windows SwitchToThread) so the +# thread a mutant waits for can run even when the runner has fewer vCPUs than runnable threads. +# TP_MUT_SPIN_MAX bounds every wait, so a mutant never hangs: it only loses the widened window. +_MUT_PRELUDE = """#include +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#define TP_MUT_YIELD() ((void)SwitchToThread()) +#else +#include +#define TP_MUT_YIELD() ((void)sched_yield()) +#endif +#define TP_MUT_SPIN_MAX 4000000L +""" + + +def _with_mut_prelude(h: str) -> str: + assert h.count("#include \n") == 1 + return h.replace("#include \n", _MUT_PRELUDE, 1) + + +def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool, hold_watch_open: bool = False) -> Path: + """A copy of tp_runtime.h with one #159 protection removed (the mutation of that protection). + + hold_watch_open (the dict-lock mutant only) makes the window deterministic. The real code is + sound because the watcher (which moves tp_globals_epoch) and the store of the new binding both + run under the dict's lock, which tp_class_global_ok also takes. Without that lock the race needs + bump < reader's epoch load < reader's lookup < store, and the writer's bump-to-store gap is a few + tens of nanoseconds: a reader gets into it only by luck, and on a 3-4 vCPU runner with seven + runnable threads it rarely does. So the mutant build itself holds the gap open: right after the + watcher moves the epoch it waits (bounded, yielding) until another thread has looked the old + binding up under the new epoch AND remembered it in rt->gepoch. Only then does the store of the + new binding go ahead. The remembered binding is then stale by construction.""" h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() g = h if drop_dict_lock: @@ -952,6 +989,22 @@ def _racy_runtime(tmp: Path, *, drop_dict_lock: bool, plain_epoch: bool) -> Path g = g.replace("tp_atomic_load_u64(&tp_globals_epoch)", "tp_globals_epoch") g = g.replace("tp_atomic_load_u64(&rt->gepoch)", "rt->gepoch") g = g.replace("tp_atomic_store_u64(&rt->gepoch, epoch)", "rt->gepoch = epoch") + if hold_watch_open: + assert drop_dict_lock and not plain_epoch + g = _with_mut_prelude(g) + decl = "static uint64_t tp_globals_epoch = 1;\n" + assert g.count(decl) == 1 + g = g.replace(decl, decl + "static uint64_t tp_mut_looked = 0; /* mutant only: last epoch a reader remembered */\n" + "#define TP_MUT_HOLD_OPEN() do { uint64_t want_ = tp_atomic_load_u64(&tp_globals_epoch); long n_; \\\n" + " for (n_ = 0; n_ < TP_MUT_SPIN_MAX && tp_atomic_load_u64(&tp_mut_looked) < want_; n_++) { \\\n" + " if ((n_ & 63) == 63) { TP_MUT_YIELD(); } } } while (0)\n") + bump = "tp_atomic_add_u64(&tp_globals_epoch, 1);\n" + assert g.count(bump) == 2 + g = g.replace(bump, "tp_atomic_add_u64(&tp_globals_epoch, 1);\n TP_MUT_HOLD_OPEN();\n") + remember = " tp_atomic_store_u64(&rt->gepoch, epoch);\n }\n return rc;" + assert g.count(remember) == 1 + g = g.replace(remember, " tp_atomic_store_u64(&rt->gepoch, epoch);\n" + " tp_atomic_store_u64(&tp_mut_looked, epoch);\n }\n return rc;") assert g != h d = tmp / "rt" d.mkdir(parents=True, exist_ok=True) @@ -964,7 +1017,7 @@ def test_mutant_without_the_dict_critical_section_is_caught(built, tmp_path): """Without the critical section around the epoch read and the global lookup, a thread can remember a binding the rebinding thread has already replaced (the watcher fires before the store).""" result, path, _ = built - rt = _racy_runtime(tmp_path, drop_dict_lock=True, plain_epoch=False) + rt = _racy_runtime(tmp_path, drop_dict_lock=True, plain_epoch=False, hold_watch_open=True) so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=rt) body = """ stop = False @@ -979,6 +1032,7 @@ def hammer(): m.Cell = m.Twin if i % 2 == 0 else orig if type(m.mk(1, 2)).__name__ != ("Twin" if i % 2 == 0 else "Cell"): bad.append(i) + break stop = True ht.join() print(json.dumps({"stale": len(bad)})) @@ -1000,15 +1054,21 @@ def test_mutant_with_a_borrowed_field_read_is_caught(built, tmp_path): h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() good = " return PyMember_GetOne((const char *)obj, m);\n" assert good in h + # The window is held open until the writer has replaced the slot (bounded, yielding), so the old + # value has lost the slot's reference before the borrowed pointer is turned into a reference: the + # mutant always resurrects a dead object (and every later release of it is a second dealloc), + # instead of relying on a writer that happens to run inside a fixed-length spin. racy = (" PyObject *v = *(PyObject *volatile *)((char *)obj + m->offset);\n" - " for (volatile int i = 0; i < 2000; i++) { }\n" + " { long n_; for (n_ = 0; n_ < TP_MUT_SPIN_MAX\n" + " && *(PyObject *volatile *)((char *)obj + m->offset) == v; n_++) {\n" + " if ((n_ & 63) == 63) { TP_MUT_YIELD(); } } }\n" " return Py_NewRef(v);\n") d = tmp_path / "rt" d.mkdir() - (d / "tp_runtime.h").write_text(h.replace(good, racy)) + (d / "tp_runtime.h").write_text(_with_mut_prelude(h.replace(good, racy))) so = cbuild.build(cgen.generate(result.module, path), "ftmod", tmp_path / "mut", runtime_dir=d) body = """ - cell = m.Cell((0, 0), None) + cell = m.Cell((0, [0]), None) stop = False bad = [] def writer(): @@ -1019,8 +1079,10 @@ def writer(): def reader(): for _ in range(20000): v = m.geta(cell) - if type(v) is not tuple or len(v) != 2: + if (type(v) is not tuple or len(v) != 2 or type(v[1]) is not list + or len(v[1]) != 1 or v[1][0] != v[0]): bad.append(1) + break wt = threading.Thread(target=writer); wt.start() errs = run_threads([reader] * 3) stop = True From e2b791002740f5706368dc30fc206888928b7157 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:21:17 +0900 Subject: [PATCH 53/61] Compose: Bind the fork's design systems in the Compose fixtures (#408) The Compose and notebook E2E fixtures depend on org.thisisthepy.compose.designsystem:designsystem{,-material3,-breeze, -cupertino,-deepin,-fluent,-gnome,-liquid-glass} at 1.11.1-ext.3 and walk their Kotlin packages, so kotlin-stubs and the notebook host carry the packages pythonx-compose maps (pythonx-compose #152). SPEC B-12, DesignSystemBindingTest and the consumer.py lines say what must hold. Blocked on the fork publishing 1.11.1-ext.3; nothing here resolves yet. --- .github/scripts/stubs/consumer.py | 5 ++ docs/SPEC.md | 13 +++ gradle/libs.versions.toml | 13 +++ .../fixtures/compose/build.gradle.kts | 20 +++++ .../compose/DesignSystemBindingTest.kt | 80 +++++++++++++++++++ .../fixtures/notebook-e2e/build.gradle.kts | 20 +++++ 6 files changed, 151 insertions(+) create mode 100644 python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt diff --git a/.github/scripts/stubs/consumer.py b/.github/scripts/stubs/consumer.py index 99e4e61d8..dd7fbaa8a 100644 --- a/.github/scripts/stubs/consumer.py +++ b/.github/scripts/stubs/consumer.py @@ -28,6 +28,7 @@ from androidx.compose.material3 import Checkbox, Icon, Text from androidx.compose.ui import Modifier from androidx.compose.ui.graphics.vector import ImageVector +from org.thisisthepy.compose.material3 import Material3DesignSystem, material_design_system # A chain started from the class, the way Kotlin writes `Modifier.padding(16.dp)`. chain: Modifier = Modifier.fillMaxWidth().padding(16.0).size(24.0) @@ -43,6 +44,10 @@ assert_type(fill_max_width(chain), Modifier) pythonic_chain: Modifier = Modifier.fill_max_width().padding(16.0).fill_max_width(0.5) +# Issue #408: the fork's design systems are walked too, under their own Kotlin packages. +dark: Material3DesignSystem = material_design_system(True) +assert_type(material_design_system(is_dark=False), Material3DesignSystem) + checked = True diff --git a/docs/SPEC.md b/docs/SPEC.md index 1950aba58..a8462a56d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -658,6 +658,19 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. in `AGENTS.md`): everything else that is public stays bound; it is not a list of what to expose. A segment must equal `internal` exactly (`internals` and `internal_api` are ordinary packages). `Status: implemented`, `GP/artifact/InternalPackageExclusionTest.kt`. +- **B-12** (#408) The Compose fixtures (`python-multiplatform-compose/fixtures/compose`, whose stubs are the + `kotlin-stubs` artefact of B-8, and `fixtures/notebook-e2e`) depend on the fork's design systems, + `org.thisisthepy.compose.designsystem:designsystem` and `designsystem-{material3, breeze, cupertino, deepin, + fluent, gnome, liquid-glass}` at `compose-design-systems` (`gradle/libs.versions.toml`, 1.11.1-ext.3), and walk + their Kotlin packages `org.thisisthepy.compose.{designsystem, material3, breeze, cupertino, deepin, fluent, + gnome, liquidglass}` under those names (decision 12.1). pythonx-compose puts `org.thisisthepy.compose.material3` + in front of `androidx.compose.material3` and exposes each design system as `pythonx.compose.` + (pythonx-compose #152; leader decision 2026-11-13 on pythonx-compose #54 that they become a dependency here once + published). The design systems build against the fork's Compose at their own version, so these two fixtures + resolve the fork's Compose at 1.11.1-ext.3 while the rest of the build stays on `compose-libraries`. + `Status: planned`, blocked on the fork publishing 1.11.1-ext.3 (compose-multiplatform-core-extended #212): + `fixtures/compose/.../DesignSystemBindingTest.kt` and the `org.thisisthepy.compose.material3` lines of + `.github/scripts/stubs/consumer.py`. ## 7. Threading and builds diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fe01e4a76..83a08cb87 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,6 +12,10 @@ androidx-material = "1.12.0" androidx-test-junit = "1.2.1" compose-plugin = "1.11.1-ext.3" compose-libraries = "1.11.1-ext.2" +# The fork's design systems (issue #408). They build against the fork's Compose at their own version, +# so a classpath that takes them resolves the fork's Compose at this version too (Gradle picks the +# highest); moving every Compose library to it is a separate change. +compose-design-systems = "1.11.1-ext.3" # JetBrains stopped publishing material-icons with Compose after 1.7.3 (Maven Central: newest is 1.7.3); # it is the version Compose Multiplatform 1.11.x points to, resolved against 1.11.1's ui/ui-graphics. compose-material-icons = "1.7.3" @@ -38,6 +42,15 @@ androidx-lifecycle-viewmodel = { group = "org.thisisthepy.compose.androidx.lifec androidx-lifecycle-runtime-compose = { group = "org.thisisthepy.compose.androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "compose-libraries" } kotlinx-coroutines-swing = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-browser = { group = "org.jetbrains.kotlinx", name = "kotlinx-browser", version.ref = "kotlinx-browser" } +# The fork's design systems (compose-multiplatform-core-extended `extended/design-systems`, issue #408). +designsystem = { group = "org.thisisthepy.compose.designsystem", name = "designsystem", version.ref = "compose-design-systems" } +designsystem-material3 = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-material3", version.ref = "compose-design-systems" } +designsystem-breeze = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-breeze", version.ref = "compose-design-systems" } +designsystem-cupertino = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-cupertino", version.ref = "compose-design-systems" } +designsystem-deepin = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-deepin", version.ref = "compose-design-systems" } +designsystem-fluent = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-fluent", version.ref = "compose-design-systems" } +designsystem-gnome = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-gnome", version.ref = "compose-design-systems" } +designsystem-liquid-glass = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-liquid-glass", version.ref = "compose-design-systems" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/python-multiplatform-compose/fixtures/compose/build.gradle.kts b/python-multiplatform-compose/fixtures/compose/build.gradle.kts index 40470cf26..64f770a11 100644 --- a/python-multiplatform-compose/fixtures/compose/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/compose/build.gradle.kts @@ -58,6 +58,17 @@ kotlin { exclude(group = "org.jetbrains.compose.runtime") exclude(group = "org.jetbrains.compose.ui") } + // The fork's design systems (issue #408): `pythonx.compose.material3` puts + // `org.thisisthepy.compose.material3` in front of `androidx.compose.material3`, and + // `pythonx.compose.` is each of these (pythonx-compose `pythonx-map.toml`). + implementation(libs.designsystem) + implementation(libs.designsystem.material3) + implementation(libs.designsystem.breeze) + implementation(libs.designsystem.cupertino) + implementation(libs.designsystem.deepin) + implementation(libs.designsystem.fluent) + implementation(libs.designsystem.gnome) + implementation(libs.designsystem.liquid.glass) implementation(compose.desktop.currentOs) } } @@ -123,6 +134,15 @@ pythonBindings { // Python calls inside a composition. Its file facade, not the package, so that the host // composables (`PythonAppView` and friends) stay out of the table. "python.multiplatform.compose.RememberSaveableKt", + // 8. The fork's design systems (issue #408), one package each. + "org.thisisthepy.compose.designsystem", + "org.thisisthepy.compose.material3", + "org.thisisthepy.compose.breeze", + "org.thisisthepy.compose.cupertino", + "org.thisisthepy.compose.deepin", + "org.thisisthepy.compose.fluent", + "org.thisisthepy.compose.gnome", + "org.thisisthepy.compose.liquidglass", ), ) // Stubs on: `generatePythonStubs` over the real Compose jars is what `.github/scripts/stubs/check-stubs.sh` (the mypy diff --git a/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt b/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt new file mode 100644 index 000000000..39c56f4df --- /dev/null +++ b/python-multiplatform-compose/fixtures/compose/src/desktopTest/kotlin/fixture/compose/DesignSystemBindingTest.kt @@ -0,0 +1,80 @@ +package fixture.compose + +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.pythonx.PythonxAdapter +import python.multiplatform.ffi.upcall.UpcallBootstrap +import python.multiplatform.generated.artifacts.ArtifactTable +import python.multiplatform.reflection.UpcallTable +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test + +/** + * SPEC B-12 (issue #408): the fork's design systems (`org.thisisthepy.compose.designsystem:*`) are + * walked, each under its own Kotlin package, so pythonx-compose's `pythonx.compose.material3` (the + * design system in front of `androidx.compose.material3`) and `pythonx.compose.` resolve. + * + * Each package must answer with the declaration that names the system; `materialDesignSystem` and + * `MaterialButton` are checked parameter by parameter, by their Kotlin names, because pythonx-compose + * calls them. Without the dependency or the walked package, the import fails. + */ +class DesignSystemBindingTest { + + @BeforeTest + fun installProducers() { + Python3.initialize(silent = true) + UpcallTable.clear() + UpcallTable.install(ArtifactTable.fragments) + check(UpcallBootstrap.publishToGlobals()) { "UpcallBootstrap.publishToGlobals() failed" } + PythonxAdapter.install(PYTHONX_RAW_VALUE_CLASSES) + } + + @AfterTest + fun cleanup() { + UpcallTable.clear() + } + + @Test + fun everyDesignSystemPackageIsBoundUnderItsKotlinName() { + Python3.exec( + """ + import importlib as _ds_il + _ds_expected = { + 'org.thisisthepy.compose.designsystem': 'LocalDesignSystem', + 'org.thisisthepy.compose.material3': 'Material3DesignSystem', + 'org.thisisthepy.compose.breeze': 'BreezeDesignSystem', + 'org.thisisthepy.compose.cupertino': 'CupertinoDesignSystem', + 'org.thisisthepy.compose.deepin': 'DeepinDesignSystem', + 'org.thisisthepy.compose.fluent': 'FluentDesignSystem', + 'org.thisisthepy.compose.gnome': 'GnomeDesignSystem', + 'org.thisisthepy.compose.liquidglass': 'LiquidGlass', + } + _ds_missing = {} + for _ds_package, _ds_name in _ds_expected.items(): + _ds_module = _ds_il.import_module(_ds_package) + if _ds_name not in dir(_ds_module): + _ds_missing[_ds_package] = sorted(n for n in dir(_ds_module) if not n.startswith('_'))[:20] + assert not _ds_missing, _ds_missing + """.trimIndent(), + ) + } + + @Test + fun theMaterialDesignSystemFunctionsCarryTheirKotlinParameters() { + Python3.exec( + """ + import python_multiplatform as _ds_pm + import org.thisisthepy.compose.material3 as _ds_m3 + + _ds_d = _ds_pm.describe(_ds_m3, 'materialDesignSystem') + assert [(_p['name'], _p['type']) for _p in _ds_d['parameters']] == \ + [('isDark', 'kotlin.Boolean')], _ds_d + + _ds_b = _ds_pm.describe(_ds_m3, 'MaterialButton') + assert [_p['name'] for _p in _ds_b['parameters']] == \ + ['variant', 'onClick', 'modifier', 'enabled', 'content'], _ds_b + assert _ds_b['parameters'][0]['type'] == 'org.thisisthepy.compose.designsystem.ButtonVariant', _ds_b + """.trimIndent(), + ) + } +} diff --git a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts index 0c269b73a..3b17746a6 100644 --- a/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts +++ b/python-multiplatform-compose/fixtures/notebook-e2e/build.gradle.kts @@ -61,6 +61,17 @@ kotlin { exclude(group = "org.jetbrains.compose.runtime") exclude(group = "org.jetbrains.compose.ui") } + // The fork's design systems (issue #408): `pythonx.compose.material3` puts + // `org.thisisthepy.compose.material3` in front of `androidx.compose.material3`, and + // `pythonx.compose.` is each of these (pythonx-compose `pythonx-map.toml`). + implementation(libs.designsystem) + implementation(libs.designsystem.material3) + implementation(libs.designsystem.breeze) + implementation(libs.designsystem.cupertino) + implementation(libs.designsystem.deepin) + implementation(libs.designsystem.fluent) + implementation(libs.designsystem.gnome) + implementation(libs.designsystem.liquid.glass) implementation(compose.desktop.currentOs) } } @@ -98,6 +109,15 @@ pythonBindings { "androidx.compose.foundation.text.input.TextFieldState", "androidx.compose.foundation.text.input.TextFieldStateKt", "python.multiplatform.compose.RememberSaveableKt", + // The fork's design systems (issue #408), one package each. + "org.thisisthepy.compose.designsystem", + "org.thisisthepy.compose.material3", + "org.thisisthepy.compose.breeze", + "org.thisisthepy.compose.cupertino", + "org.thisisthepy.compose.deepin", + "org.thisisthepy.compose.fluent", + "org.thisisthepy.compose.gnome", + "org.thisisthepy.compose.liquidglass", ), ) } From ee1ee7ce690a8e9c6b1591aea457eacd462a34a6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:30:57 +0900 Subject: [PATCH 54/61] CI: probe embedding the 3.15t wasm build without main (#175) Exports wasmExports, wasmMemory and wasmTable from the free-threaded wasm build and adds a node probe step to build-ft-wasm.yml: the shared memory, a non-shared versus shared memory import (what Kotlin/Wasm emits versus what a patch would emit), and Py_Initialize from the JS main thread with four threads and gc.collect() in a worker. The step runs after the archive upload, so a probe failure does not lose the archive. --- .github/scripts/ft-wasm/build-wasm.sh | 54 ++++++++ .github/scripts/ft-wasm/embed-probe.mjs | 167 ++++++++++++++++++++++++ .github/workflows/build-ft-wasm.yml | 21 +++ docs/platforms/free-threaded-wasm.md | 38 ++++++ 4 files changed, 280 insertions(+) create mode 100644 .github/scripts/ft-wasm/embed-probe.mjs diff --git a/.github/scripts/ft-wasm/build-wasm.sh b/.github/scripts/ft-wasm/build-wasm.sh index 328861309..5e9e86cff 100755 --- a/.github/scripts/ft-wasm/build-wasm.sh +++ b/.github/scripts/ft-wasm/build-wasm.sh @@ -150,6 +150,59 @@ grep -Eq '^CONFIGURE_LDFLAGS_NODIST=.*-sUSE_PTHREADS' "$hostdir/Makefile" \ grep -Eq '^ABIFLAGS=[[:space:]]*t$' "$hostdir/Makefile" || die "host Makefile ABIFLAGS is not t" log "ok: host configured with --disable-gil --enable-wasm-pthreads, ABIFLAGS=t" +# ---- patch: JS reachability for the embedding probe ----------------------- +# Same substitution as patch_makefile_reachability in python-multiplatform/scripts/wasm/build-cpython.sh: +# the library reaches CPython without main through Module.wasmExports / Module.wasmMemory, which +# the stock link line does not expose (3.15 default: FS,callMain,ENV,HEAPU32,TTY). Written as a +# regex over the whole comma list rather than a fixed string, because the 3.15 Makefile was not +# available when this was written. The result is asserted, so a changed link line fails the +# build here instead of producing an archive the probe cannot use. +# The C functions: CPython's LINKFORSHARED lists -sEXPORTED_FUNCTIONS explicitly +# (3.14: _main,_Py_Version,__PyRuntime,...) next to -sMAIN_MODULE. Whether 3.15 still does is +# unverified, so _malloc,_free,_Py_Version are appended to that list when it exists (harmless if +# MAIN_MODULE already exports everything); the probe then checks the exports it needs and +# fails loudly if one is missing. +patch_makefile_exports() { + python3 - "$hostdir/Makefile" <<'PATCH' +import re +import sys +from pathlib import Path +p = Path(sys.argv[1]); t = p.read_text() + +def add_runtime_methods(m): + items = m.group(2).split(",") + for extra in ("wasmExports", "wasmMemory", "wasmTable"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, n = re.subn(r"(-sEXPORTED_RUNTIME_METHODS=)(\S+)", add_runtime_methods, t) +assert n >= 1, "no -sEXPORTED_RUNTIME_METHODS= in the host Makefile; the link line changed shape, re-check the patch" +for m in re.finditer(r"-sEXPORTED_RUNTIME_METHODS=(\S+)", t): + methods = m.group(1).split(",") + assert all(x in methods for x in ("wasmExports", "wasmMemory", "wasmTable")), "wasmExports/wasmMemory/wasmTable missing after the patch: " + m.group(0) +print(f"Makefile: wasmExports,wasmMemory,wasmTable added to {n} -sEXPORTED_RUNTIME_METHODS list(s)") + +def add_functions(m): + items = m.group(2).split(",") + for extra in ("_malloc", "_free", "_Py_Version"): + if extra not in items: + items.append(extra) + return m.group(1) + ",".join(items) + +t, k = re.subn(r"(-sEXPORTED_FUNCTIONS=)(\S+)", add_functions, t) +if k: + print(f"Makefile: _malloc,_free,_Py_Version ensured in {k} -sEXPORTED_FUNCTIONS list(s)") +else: + print("Makefile: NOTE no -sEXPORTED_FUNCTIONS= found; relying on the link exporting C symbols another way (the probe checks)") +p.write_text(t) +for line in t.splitlines(): + if "-sEXPORTED_RUNTIME_METHODS" in line or "-sEXPORTED_FUNCTIONS" in line or "-sMAIN_MODULE" in line: + print("Makefile link line:", line.strip()[:2000]) +PATCH +} +patch_makefile_exports + # ---- host build ----------------------------------------------------------- # make-host also runs `python.sh --version`, so a startup failure (for example the trampoline # instantiation above, with FT_WASM_TRAMPOLINE_PATCH=skip) fails this step. @@ -222,6 +275,7 @@ cp "$hostdir/pyconfig.h" "$stage/include/$stdlib_name/" echo "CFLAGS=$HOST_CFLAGS" echo "LDFLAGS=$HOST_LDFLAGS" echo "trampoline patch: $TRAMPOLINE_PATCH" + echo "link patch: wasmExports,wasmMemory,wasmTable in EXPORTED_RUNTIME_METHODS (embedding probe)" echo "built from https://www.python.org/ftp/python/ (Python-$PY_VERSION.tgz)" } > "$stage/BUILD-INFO.txt" diff --git a/.github/scripts/ft-wasm/embed-probe.mjs b/.github/scripts/ft-wasm/embed-probe.mjs new file mode 100644 index 000000000..375d22396 --- /dev/null +++ b/.github/scripts/ft-wasm/embed-probe.mjs @@ -0,0 +1,167 @@ +// Embedding probe for the free-threaded wasm32-emscripten archive (#175 step 2, agent proposal). +// +// Usage: node embed-probe.mjs +// +// Reaches CPython the way python-multiplatform does: python.mjs is loaded as a module factory +// with noInitialRun (main is never called), and everything goes through Module.wasmExports on the +// JS main thread. Prints one line per check, "[x] PASS|FAIL: detail", and a final +// FT-WASM-EMBED-PROBE-OK only when every check passed. A hang is not caught here; the caller +// wraps this in `timeout` and reads the last "[x]" line it printed. +// +// Node only. Node's main thread uses a real Atomics.wait; a browser's main thread busy-waits and +// services proxied calls, so this probe says nothing about the browser. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const root = path.resolve(process.argv[2] ?? ""); +if (!process.argv[2] || !fs.existsSync(path.join(root, "python.mjs"))) { + console.log(`[setup] FAIL: usage: node embed-probe.mjs , got '${process.argv[2]}'`); + process.exit(2); +} + +let failed = false; +const pass = (id, detail) => console.log(`[${id}] PASS: ${detail}`); +const fail = (id, detail) => { failed = true; console.log(`[${id}] FAIL: ${detail}`); }; +const finish = () => { + if (failed) { console.log("FT-WASM-EMBED-PROBE-FAILED"); process.exit(1); } + console.log("FT-WASM-EMBED-PROBE-OK"); + process.exit(0); // the pthread Workers would otherwise keep the process alive +}; + +// Same reason as cpython.mjs: with no main there is no WebAssembly.promising frame, so a +// JSPI-suspending syscall import would throw SuspendError. +delete WebAssembly.promising; +delete WebAssembly.Suspending; + +const libDir = fs.readdirSync(path.join(root, "lib")).find((d) => /^python3\.\d+t$/.test(d)); +if (!libDir) { fail("a", `no lib/python3.t in ${root}`); finish(); } +const minor = libDir.replace(/^python/, ""); + +// ---- a. load the factory without main ----------------------------------------------------------- +let M; +try { + const { default: EmscriptenModule } = await import(pathToFileURL(path.join(root, "python.mjs")).href); + M = await EmscriptenModule({ + noInitialRun: true, + thisProgram: `${root}/bin/python${minor}`, + arguments: [], + preRun(Module) { + globalThis.Module = Module; + // Minimum of node_entry.mjs: mount the host filesystem (the archive is read through + // NODEFS) except the three directories Emscripten owns, then set the environment. + for (const dir of fs.readdirSync("/").filter((d) => !["dev", "lib", "proc"].includes(d)).map((d) => "/" + d)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } + Module.FS.chdir(root); + Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; + Module.ENV.PYTHONHOME = root; + }, + }); + const mem = M.wasmMemory; + if (!mem) throw new Error("Module.wasmMemory is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!M.wasmExports) throw new Error("Module.wasmExports is undefined (the EXPORTED_RUNTIME_METHODS patch did not take effect)"); + if (!(mem.buffer instanceof SharedArrayBuffer)) throw new Error("wasmMemory.buffer is not a SharedArrayBuffer"); + pass("a", `wasmMemory.buffer is a SharedArrayBuffer, ${mem.buffer.byteLength / 65536} pages`); +} catch (e) { + fail("a", String(e && e.stack || e)); + finish(); +} +const E = M.wasmExports; +// A fresh view each time: a shared memory's old buffer keeps its old length after growth. +const u8 = () => new Uint8Array(M.wasmMemory.buffer); +const u32 = () => new Uint32Array(M.wasmMemory.buffer); +const exported = (name) => { + let v = E[name]; + if (v === undefined) v = M["_" + name]; + if (v === undefined) throw new Error(`CPython export '${name}' not found on wasmExports or Module`); + return v; +}; +const addrOf = (name) => { const v = exported(name); return v instanceof WebAssembly.Global ? v.value : v; }; + +// ---- b. memory import checks A1 / A2 ------------------------------------------------------------ +// Hand-assembled modules importing intrinsics.memory (what Kotlin/Wasm's import is named) and +// exporting load(i32) -> i32 = i32.load. A1 imports a non-shared memory (what Kotlin 2.4.20-Beta2 +// emits); A2 a shared one with maximum 65536 pages (what a post-link patch would emit). +const head = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; +const typeSec = [0x01, 0x06, 0x01, 0x60, 0x01, 0x7f, 0x01, 0x7f]; +const importPrefix = [0x01, 0x0a, ...Buffer.from("intrinsics"), 0x06, ...Buffer.from("memory"), 0x02]; +const tail = [0x03, 0x02, 0x01, 0x00, 0x07, 0x08, 0x01, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x00, 0x00, + 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x28, 0x02, 0x00, 0x0b]; +const mod = (limits) => new Uint8Array([...head, ...typeSec, + 0x02, importPrefix.length + limits.length, ...importPrefix, ...limits, ...tail]); +const A1 = mod([0x00, 0x00]); +const A2 = mod([0x03, 0x00, 0x80, 0x80, 0x04]); +try { + let a1; + try { + new WebAssembly.Instance(new WebAssembly.Module(A1), { intrinsics: { memory: M.wasmMemory } }); + a1 = "instantiated"; + } catch (e) { + a1 = e instanceof WebAssembly.LinkError ? "LinkError" : `other:${e && e.name}`; + if (a1 === "LinkError") console.log(` A1 message: ${e.message}`); + } + if (a1 !== "LinkError") throw new Error(`A1 (non-shared import) expected LinkError, got ${a1}`); + const a2 = new WebAssembly.Instance(new WebAssembly.Module(A2), { intrinsics: { memory: M.wasmMemory } }); + const addr = addrOf("Py_Version"); + const viaLoad = a2.exports.load(addr) >>> 0; + const viaHeap = u32()[addr >>> 2]; + if (viaLoad !== viaHeap) throw new Error(`A2.load(${addr}) = ${viaLoad} but HEAPU32 = ${viaHeap}`); + pass("b", `A1 non-shared import: LinkError; A2 shared import (max 65536): instantiated; A2.load(&Py_Version=${addr}) = 0x${viaLoad.toString(16)} equals HEAPU32`); +} catch (e) { + fail("b", String(e && e.stack || e)); +} + +// ---- c. embedding without main, threads, gc in a worker ----------------------------------------- +const cstr = (s) => { + const bytes = Buffer.from(s + "\0"); + const p = E.malloc(bytes.length); + if (!p) throw new Error("malloc returned 0"); + u8().set(bytes, p); + return p; +}; +const script = ` +import gc, os, sys, threading +assert sys._is_gil_enabled() is False, "GIL is enabled at the start" +sizes = {} +def reader(i): + with open(os.__file__) as f: + n = len(f.read()) + print(f"embed-probe thread {i} read {n} chars of os.py", flush=True) + sizes[i] = n +threads = [threading.Thread(target=reader, args=(i,)) for i in range(4)] +for t in threads: + t.start() +for t in threads: + t.join(timeout=120) +alive = [t.name for t in threads if t.is_alive()] +assert not alive, f"threads still running: {alive}" +assert len(sizes) == 4 and len(set(sizes.values())) == 1, sizes +assert sys._is_gil_enabled() is False, "GIL was re-enabled by the threads" +collected = [] +gc_thread = threading.Thread(target=lambda: collected.append(gc.collect())) +gc_thread.start() +gc_thread.join(timeout=120) +assert not gc_thread.is_alive() and collected, "gc.collect() in a worker thread did not finish" +print(f"embed-probe gc.collect() in a worker thread returned {collected[0]}", flush=True) +assert sys._is_gil_enabled() is False, "GIL was re-enabled by gc" +`; +try { + E.Py_Initialize(); + pass("c1", "Py_Initialize returned"); + const tstate = E.PyEval_SaveThread(); + pass("c2", `PyEval_SaveThread returned ${tstate} (main thread detached)`); + const gil = E.PyGILState_Ensure(); + const code = cstr(script); + console.log(" running the thread script (a hang here is caught by the caller's timeout)"); + const rc = E.PyRun_SimpleString(code); + E.free(code); + E.PyGILState_Release(gil); + if (rc !== 0) throw new Error(`PyRun_SimpleString returned ${rc} (Python traceback, if any, is above)`); + pass("c3", "4 threads read os.py, joined, GIL stayed off, gc.collect() ran in a worker thread; PyGILState_Release returned"); +} catch (e) { + fail("c", String(e && e.stack || e)); +} +finish(); diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml index b0242b7e9..6baebdbe6 100644 --- a/.github/workflows/build-ft-wasm.yml +++ b/.github/workflows/build-ft-wasm.yml @@ -83,3 +83,24 @@ jobs: name: python-${{ inputs.python_version || '3.15.0' }}t-wasm32-emscripten path: ${{ env.FT_OUT }}/*.tar.gz if-no-files-found: error + + # Step 2 of #175 (agent proposal): embed CPython without main from the node main thread, as + # the library does. A separate step after the upload, so a probe failure or hang does not + # lose the archive. Node only: node uses a real Atomics.wait on the main thread, a browser + # busy-waits, so this says nothing about the browser. + - name: Embedding probe (no main) + run: | + root="$(ls -d "$FT_WORK"/wasm-check/python-*t-wasm32-emscripten | head -n 1)" + [ -d "$root" ] || { echo "no unpacked archive under $FT_WORK/wasm-check"; exit 1; } + log="$FT_WORK/embed-probe.log" + set +e + timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs "$root" > "$log" 2>&1 + rc=$? + set -e + cat "$log" + if [ "$rc" -ne 0 ]; then + last="$(grep -E '^\[[a-z0-9]+\]' "$log" | tail -n 1)" + echo "::error::embedding probe exited $rc (124 = timed out, a hang); last check printed: ${last:-none}" + exit "$rc" + fi + grep -q '^FT-WASM-EMBED-PROBE-OK$' "$log" diff --git a/docs/platforms/free-threaded-wasm.md b/docs/platforms/free-threaded-wasm.md index aab98375c..c3fe705dc 100644 --- a/docs/platforms/free-threaded-wasm.md +++ b/docs/platforms/free-threaded-wasm.md @@ -117,6 +117,44 @@ The script stops at the first failed check, in this order: The archive is uploaded as the workflow artifact `python-t-wasm32-emscripten`. No release is published or changed. +## Embedding probe (step 2 of #175) + +Agent proposal toward #175 step 2, not a decision. It answers, on CI, whether the library's way +of using CPython (load `python.mjs` as a factory with `noInitialRun`, never call `main`, call +exports from the JS main thread) works against this pthreads build. + +`build-wasm.sh` now appends `wasmExports,wasmMemory,wasmTable` to every +`-sEXPORTED_RUNTIME_METHODS` list in the host Makefile before `make-host` (the same edit as +`patch_makefile_reachability` in `python-multiplatform/scripts/wasm/build-cpython.sh`, as a +regex because the 3.15 Makefile text was not checked) and asserts the result. If the Makefile +has a `-sEXPORTED_FUNCTIONS=` list it also ensures `_malloc,_free,_Py_Version` are in it. A +missing list is only logged: whether 3.15 still exports C symbols through `-sMAIN_MODULE` +(as 3.14 does) is unverified, and the probe fails loudly if an export it needs is absent. + +The workflow step `Embedding probe (no main)` runs after the archive upload, so a probe +failure or hang does not lose the archive: +`timeout 600 node .github/scripts/ft-wasm/embed-probe.mjs `. It prints one +`[x] PASS|FAIL` line per check and `FT-WASM-EMBED-PROBE-OK` only if all pass; on a timeout +(exit 124) the step reports the last check line printed. + +- `a`: the factory loads with `noInitialRun`, `Module.wasmMemory.buffer` is a + `SharedArrayBuffer` (page count printed). +- `b`: two hand-assembled modules import `intrinsics.memory` (the name Kotlin/Wasm uses). + A1 imports a non-shared memory and must fail with `LinkError`; A2 imports a shared one with + maximum 65536 pages and must instantiate, and `A2.load(&Py_Version)` must equal the value + read through a view of `wasmMemory`. A2 passing keeps the post-link import patch (investigation + options 2a/2b) open; A1 not failing would mean the blocker is not real. +- `c`: `Py_Initialize`, `PyEval_SaveThread`, then `PyGILState_Ensure` and `PyRun_SimpleString` + (strings placed with the exported `malloc`) of a script that starts four threads which print + and read `os.py`, joins them, asserts `sys._is_gil_enabled()` is False, and runs + `gc.collect()` inside a worker thread. This exercises the main thread blocking in `join` + while Workers need it to service proxied syscalls. + +Node only. Node's main thread uses a real `Atomics.wait`; a browser's main thread busy-waits +and services proxied calls, so a pass here does not show the browser works, and a Node deadlock +may not occur in a browser. It does not involve Kotlin at all (no upcalls, no proxies, no +Kotlin-side memory import). + ## Not verified - The workflow has not run yet. Whether 3.15.0 configures, compiles and links with From 156cb660a5b408eed75d121c8751e9276c5b3576 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:32:44 +0900 Subject: [PATCH 55/61] TypedPython: read the Python frame depth once per wrapper call (#125) The Python frames below a compiled call cannot change while it runs, so tp_python_depth's frame walk moves to the first depth check of each wrapper call (tp_depth_seg_begin/end reset it around the impl call, so a nested wrapper's return re-measures exactly where the old cache swapped). Py_GetRecursionLimit and Py_EnterRecursiveCall stay per frame: a __del__ can change the limit mid-call, and 3.15's check is a C-stack check. Design 4.3.3b. --- docs/design/typedpython.md | 47 ++ .../src/main/python/typedpython/cgen.py | 13 +- .../main/python/typedpython/runtime/API.md | 15 + .../python/typedpython/runtime/tp_runtime.h | 52 +- .../src/test/python/test_depth_once.py | 546 ++++++++++++++++++ 5 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_depth_once.py diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 686ebc722..cf583897b 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -267,6 +267,53 @@ pypackpack `BundleRequest` 로 옮긴다. pypackpack 은 TypedPython 을 실행 프레임이 끼면 그 프레임은 세지 않는다. 테스트는 두 경우 모두 "둘 다 예외를 내고 죽지 않는다" 만 단언하고 깊이를 기록한다(`test_recursion.py`). +### 4.3.3b 파이썬 깊이는 구간마다 한 번 잰다 (#125, 에이전트 제안, 확인되지 않음) + +4.3.3a 의 진입 검사는 컴파일 프레임마다 세 가지를 부른다: `PyThreadState_GetFrame` (`tp_python_depth`, +최상위 프레임 동일성 캐시를 위해), `Py_GetRecursionLimit`, `Py_EnterRecursiveCall`. 출처는 CPython +v3.15.0rc1 태그다. + +- **파이썬 깊이: 옮긴다.** wrapper 의 impl 호출 하나를 *구간* 으로 본다. 구간 안에서 컴파일 프레임 밑의 + 파이썬 프레임은 바뀌지 않는다. 사이에 도는 파이썬 코드(콜백, `__del__`, 시그널 처리기, redo)는 쌓은 + 프레임을 모두 걷고 C 로 돌아오고, 그 밑의 프레임은 위에서 C 가 도는 동안 끝날 수 없다. + `PyThreadState_GetFrame` (`Python/pystate.c:2120`) 이 보는 `_PyThreadState_GetFrame` + (`Include/internal/pycore_interpframe.h:280`, 미완성 프레임을 건너뜀)과 `PyFrame_GetBack` + (`Objects/frameobject.c:2398`) 의 결과도 그래서 같다. 구간의 첫 검사가 재고(`tp_seg_depth`), 이후 검사는 + 그 값을 쓴다. wrapper 는 impl 호출 바로 앞과 바로 뒤에서 값을 지운다(`tp_depth_seg_begin` / `_end`). + 앞의 리셋이 없으면 안쪽 wrapper 가 바깥 구간의 얕은 깊이를 쓰고, 뒤의 리셋이 없으면 바깥 구간이 안쪽의 + 깊은 깊이를 계속 쓴다. 뒤의 리셋 덕에 바깥 구간은 안쪽 wrapper 가 돌아온 뒤 다음 검사에서 다시 재는데, + 이것은 예전에 `tp_python_depth` 가 다른 최상위 프레임을 보고 캐시를 바꾸던 바로 그 지점이다. 그래서 + 캐시가 붙잡는 프레임 객체와 그 수명(아래)도 그대로다. 검사 값은 예전 값과 같은 수이므로 + RecursionError 가 나는 깊이와 프레임은 바뀌지 않는다. 스레드마다 `tp_tls` 에 있고, `count` 처럼 한 + 스레드의 C 스택에서 호출이 LIFO 로 포개진다고 가정한다(greenlet 처럼 C 스택을 바꾸는 경우는 `count` 도 + 이미 맞지 않는다). +- **재귀 한도: 옮기지 않는다.** `Py_GetRecursionLimit` (`Python/ceval.c:7`) 은 interp 의 값을 읽는다. + `Py_SetRecursionLimit` (`Python/ceval.c:14`) 는 stop-the-world 로 바꾸지만, 같은 스레드에서는 컴파일 코드의 + 어느 `Py_DECREF` 든 `__del__` 을, 어느 호출이든 콜백을 돌릴 수 있고 거기서 `sys.setrecursionlimit` 을 + 부를 수 있다. 인터프리터는 다음 프레임에서 새 한도를 본다(`_Py_EnterRecursivePy`, + `Python/ceval_macros.h:406`, `py_recursion_remaining` 은 `Py_SetRecursionLimit` 이 고친다). 바뀐 것을 알 공개 + 수단이 없으므로 프레임마다 읽는다. +- **C 스택 가드: 옮기지 않는다.** 3.15 의 `Py_EnterRecursiveCall` (`Python/ceval.c:3576` → + `_Py_EnterRecursiveCallTstate`, `Include/internal/pycore_ceval.h:233`) 은 계수기가 아니라 이 프레임의 스택 + 주소를 `c_stack_soft_limit` 와 비교한다(`_Py_MakeRecCheck`, 같은 파일 211행). 깊은 프레임일수록 주소가 + 다르므로 한 번만 부르면 한도를 올린 깊은 재귀가 C 스택을 넘는다. `Py_LeaveRecursiveCall` 은 빈 함수다 + (같은 파일 256행). + +**관측되지 않는 근거와 시험.** `python-multiplatform-ksp/src/test/python/test_depth_once.py`. 구현 전 빨강: +생성 C 구조 시험 둘(wrapper 의 리셋, 헤더의 `tp_seg_depth`; 2026-11-13 cgen 을 일반 Python 으로 돌려 관측), +`-DTP_TRACE_DEPTH` 의 `__tp_trace_frame_reads__` 가 `down(100)` 에서 1 (예전 101, 추정). 구현 전에도 +초록인 보존 시험: 한도 주변 모든 파이썬 깊이와 재귀 깊이에서 인터프리터와 비교, 콜백 안의 중첩 컴파일 +호출(안쪽과 바깥쪽), 재귀 도중 `__del__` 과 콜백이 한도를 낮추고 올리기, 4 스레드, `sys._getframe` 이 보는 +프레임, 프레임 수명. 변이 셋: 앞 리셋 없음, 뒤 리셋 없음, 한도를 첫 프레임에서만 읽기. 빌드와 실행은 CI +에서 한다(에이전트는 빌드하지 않는다). + +**기존의 차이 둘(이 변경이 만든 것이 아니고 바꾸지도 않는다, 확인할 점).** (1) 컴파일 프레임 위에서 +돌아가는 순수 파이썬 프레임은 `py_recursion_remaining` 만으로 검사되므로 컴파일 프레임을 세지 않는다. 그 +위치에서 한도에 닿으면 인터프리터보다 늦게 실패한다. (2) 캐시가 안쪽 구간의 최상위 프레임(콜백 g)을 강한 +참조로 쥐므로, g 가 돌아올 때 프레임 객체가 지역 변수를 넘겨받고(`take_ownership`, +`Python/frame.c:48`, `:119`) 그 지역은 바깥 구간이 다시 잴 때 죽는다. 인터프리터에서는 g 가 돌아올 때 +죽는다. 시험은 이 시점을 develop 그대로 고정한다. + ### 4.3.4 객체 비용: 가상 객체, refcount 생략, 영역 할당 (SPEC N-12, 설계) 사용자 지시 (2026-10-03): "Rust 쪽으로 더 가는 방법을 적용하되 cpython과 동일성을 보장할 방법을 찾아." diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 10baaa1a2..ca490fe6b 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -60,7 +60,10 @@ Depth (issue #57): every impl function calls `tp_enter_call()` before it owns anything (a refusal returns -1 directly: RecursionError set, nothing counted) and `tp_leave_call()` in the one exit -block, so ok, error and deopt exits all leave exactly once. +block, so ok, error and deopt exits all leave exactly once. The wrapper brackets its impl call with +`tp_depth_seg_begin` / `tp_depth_seg_end` (#125): the Python depth is measured at the first check of +that call and reused by every later check until a wrapper of this module returns inside it +(tp_runtime.h "Segments"); the recursion limit and the C-stack guard are still read per frame. Fast entry (issue #147). A *bounded* function (`_bounded_functions`: no call cycle, every callee bounded, no node that can run Python code) has a call tree of at most k compiled frames. Its @@ -3114,7 +3117,10 @@ def wrapper(self) -> str: lock = "&tp_lock" if lst else "NULL" L.append(f" tp_osr tp_ox = {{{lock}, {{{l0}, {l1}}}, NULL}};") L.append(" tp_t->osr = &tp_ox; /* #152: taken by the impl at entry */") + # #125: the impl call is one depth segment (tp_runtime.h "Segments"): reset before and after + L.append(" tp_depth_seg_begin(tp_t);") L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module, tp_ts, tp_t{''.join(', ' + a for a in args)});") + L.append(" tp_depth_seg_end(tp_t);") if osr: L.append(" tp_t->osr = NULL;") L.append(" if (tp_s == 2) { /* left for the continuation: its result, lists already final */") @@ -3633,6 +3639,11 @@ def _exec_function(mg: _ModGen) -> str: " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_fast_hits__\", tp_cf) < 0) goto tp_done;", " tp_release(&tp_cf);", "#endif", + "#ifdef TP_TRACE_DEPTH", + " tp_cf = PyCFunction_NewEx(&tp_trace_frame_reads_def, tp_module, NULL);", + " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__tp_trace_frame_reads__\", tp_cf) < 0) goto tp_done;", + " tp_release(&tp_cf);", + "#endif", # SPEC N-14: strong references to the originals' code objects, read by tp_traced at every entry f" for (tp_k = 0; tp_k < {len(mg.names)}; tp_k++) {{", " tp_k2 = PyDict_GetItemRef(tp_table, tp_st->names[tp_k], &tp_f);", diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 4deb3819b..c37e4d253 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -270,6 +270,21 @@ per module (the header is `static`) and per thread (`TP_THREAD_LOCAL`): frames o compiled module that sit between two frames of this one are not counted, and another thread's frames never are. + Py_ssize_t tp_seg_depth(PyThreadState *ts, tp_tls *t); /* the segment's Python depth: measured (tp_python_depth) at its first check, then reused */ + void tp_depth_seg_begin(tp_tls *t); /* seg = -1; the wrapper, right before its impl call */ + void tp_depth_seg_end(tp_tls *t); /* seg = -1; the wrapper, right after its impl call */ +Segments (#125, agent proposal, not confirmed): `tp_enter_call` and `tp_depth_room` take the Python +depth from `tp_seg_depth`, so one impl call from its wrapper reads the top frame +(`PyThreadState_GetFrame`) once, not once per compiled frame. Sound because the Python frames below +compiled frames cannot change while they run: Python code started from compiled code (callback, +`__del__`, signal handler, redo) pops every frame it pushed before it returns. The wrapper's two +resets make a nested wrapper measure its own depth and make the outer segment measure again at its +next check after a nested wrapper returned, which is where `tp_python_depth` would have found a +different top frame before; the frame cache therefore changes at exactly the same points. +`Py_GetRecursionLimit()` and `Py_EnterRecursiveCall` stay per frame (a `__del__` can change the limit +between two frames; the C-stack guard compares each frame's stack address). `-DTP_TRACE_DEPTH` +(tests only) adds `__tp_trace_frame_reads__()`: the number of top-frame reads. + ## Fast entry (bounded functions; issue #147) int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k); /* 1 iff tp_python_depth(ts) + count + k <= Py_GetRecursionLimit() */ void tp_trace_fast_hit(void); /* counts a fast entry; a no-op unless -DTP_TRACE_FAST */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 37f4b8a2f..9bbf20d78 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -2862,6 +2862,7 @@ typedef struct { Py_ssize_t count; /* compiled frames in flight on this thread (this module) */ PyObject *frame; /* strong ref: the top Python frame when `pydepth` was measured */ Py_ssize_t pydepth; /* Python frames below and including `frame` */ + Py_ssize_t seg; /* Python depth of the current segment, -1 until measured (tp_seg_depth) */ } tp_depth_state; #ifndef TP_POLL_INTERVAL @@ -2927,17 +2928,26 @@ typedef struct { tp_osr *osr; /* the OSR context for the impl about to be entered (#152), or NULL */ } tp_tls; -static TP_THREAD_LOCAL tp_tls tp_tls_v = {{0, NULL, 0}, TP_POLL_INTERVAL, 0, NULL}; +static TP_THREAD_LOCAL tp_tls tp_tls_v = {{0, NULL, 0, -1}, TP_POLL_INTERVAL, 0, NULL}; static inline tp_tls *tp_tls_get(void) { return &tp_tls_v; } +#ifdef TP_TRACE_DEPTH +/* Test builds only (-DTP_TRACE_DEPTH): how many times tp_python_depth read the top frame. */ +static Py_ssize_t tp_trace_frame_reads_count = 0; +#define TP_TRACE_FRAME_READ() (tp_trace_frame_reads_count++) +#else +#define TP_TRACE_FRAME_READ() ((void)0) +#endif + /* Python frame depth of the running thread: public API only. The walk is O(depth), so it is * cached by the identity of the top frame (kept alive while compiled frames are in flight). */ static inline Py_ssize_t tp_python_depth(PyThreadState *ts, tp_tls *t) { + TP_TRACE_FRAME_READ(); PyFrameObject *top = PyThreadState_GetFrame(ts); /* new ref or NULL */ if (top == NULL) { return 0; @@ -2962,9 +2972,35 @@ static inline Py_ssize_t tp_python_depth(PyThreadState *ts, tp_tls *t) return depth; } +/* Segments (#125, design 4.3.3b, agent proposal, not confirmed). A segment is one impl call made + * by its wrapper, up to the return of any wrapper of this module called inside it. Within a + * segment the Python frames below the compiled frames do not change: whatever Python code runs in + * between (a callback, a __del__, a signal handler, a redo) pushes frames and pops every one of them + * before it returns to C, and the frames below cannot return while C above them runs. So the first + * depth check of a segment measures (tp_python_depth: one PyThreadState_GetFrame, and the walk when + * the top frame changed) and every later check reuses `seg`: exactly the value tp_python_depth would + * return there, with the frame cache in exactly the state it would leave. The wrapper resets `seg` + * right before its impl call (a nested wrapper must not take the outer segment's depth) and right + * after it (the outer segment then measures again at its next check, where it measured before + * segments existed: the cached frame, and so the lifetime of a returned callee's frame object, is + * unchanged). Not moved, per frame as before: Py_GetRecursionLimit (a __del__ or a callback can call + * sys.setrecursionlimit between two frames) and Py_EnterRecursiveCall (the C-stack guard compares + * this frame's stack address). Thread-local like `count`; both assume calls nest in LIFO order on a + * thread's C stack. */ +static inline Py_ssize_t tp_seg_depth(PyThreadState *ts, tp_tls *t) +{ + if (t->depth.seg < 0) { + t->depth.seg = tp_python_depth(ts, t); + } + return t->depth.seg; +} + +static inline void tp_depth_seg_begin(tp_tls *t) { t->depth.seg = -1; } +static inline void tp_depth_seg_end(tp_tls *t) { t->depth.seg = -1; } + static inline int tp_enter_call(PyThreadState *ts, tp_tls *t) { - if (tp_python_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) { + if (tp_seg_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) { PyErr_SetString(PyExc_RecursionError, "maximum recursion depth exceeded"); return -1; } @@ -3003,7 +3039,7 @@ static inline void tp_leave_call(tp_tls *t) * compiled calls started from it see the true depth. */ static inline int tp_depth_room(PyThreadState *ts, tp_tls *t, Py_ssize_t k) { - return tp_python_depth(ts, t) + t->depth.count + k <= Py_GetRecursionLimit(); + return tp_seg_depth(ts, t) + t->depth.count + k <= Py_GetRecursionLimit(); } /* After a fast tree: tp_python_depth may have cached the top frame (a strong reference) and, with @@ -3042,6 +3078,16 @@ static PyObject *tp_trace_fast_hits(PyObject *tp_module, PyObject *tp_unused) static inline void tp_trace_fast_hit(void) {} #endif +#ifdef TP_TRACE_DEPTH +/* Test builds only: `__tp_trace_frame_reads__()`, installed by the exec slot. */ +static PyObject *tp_trace_frame_reads(PyObject *tp_module, PyObject *tp_unused) +{ + (void)tp_module; (void)tp_unused; + return PyLong_FromSsize_t(tp_trace_frame_reads_count); +} +static PyMethodDef tp_trace_frame_reads_def = {"__tp_trace_frame_reads__", tp_trace_frame_reads, METH_NOARGS, NULL}; +#endif + /* ------------------------------------------------------------------------------------------ * Eval breaker (issue #141). The interpreter checks between bytecodes for signals, pending calls, * GIL drop requests, async exceptions and scheduled collections; compiled code has no bytecodes. diff --git a/python-multiplatform-ksp/src/test/python/test_depth_once.py b/python-multiplatform-ksp/src/test/python/test_depth_once.py new file mode 100644 index 000000000..64e02af2b --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_depth_once.py @@ -0,0 +1,546 @@ +"""TypedPython: the Python frame depth is measured once per segment, not once per compiled frame +(#125, docs/design/typedpython.md 4.3.3b, agent proposal, not confirmed). + +A *segment* is one call of a compiled impl from its wrapper (`tp_wrap_f`), up to the first point +where a call of a wrapper of the same module returns inside it. Inside a segment the Python frames +below the compiled frames cannot change: Python code that runs in between (a callback, a `__del__`, +a signal handler) pushes frames and pops all of them before it returns to C. So the first depth check +of a segment measures the Python depth (`tp_python_depth`, one `PyThreadState_GetFrame`) and every +later check of the segment reuses it. The wrapper resets the measurement before and after its impl +call, so a nested wrapper never sees the outer segment's depth and the outer segment measures again +after a nested wrapper returned (exactly where it measured before this change, so the cached frame +object and every frame's lifetime stay the same). + +What must NOT move, and stays per frame: `Py_GetRecursionLimit()` (a `__del__` or a callback can call +`sys.setrecursionlimit` between two compiled frames) and `Py_EnterRecursiveCall` (the C-stack guard: +it compares this frame's stack address). + + * red before the implementation: `__tp_trace_frame_reads__` (-DTP_TRACE_DEPTH) counts one + `PyThreadState_GetFrame` per compiled frame (`down(100)`: 101 instead of 1), and the generated + wrappers have no `tp_depth_seg_begin` / `tp_depth_seg_end`; + * green before and after (they pin what must not change): every differential test against the + interpreter (depth at every Python depth and recursion depth around the limit, a nested compiled + call from a callback, the limit changed by a `__del__` and by a callback mid-recursion, threads), + the frame-lifetime test and the `sys._getframe` test; + * mutants: a wrapper that does not reset before its impl, one that does not reset after it, and a + limit read once per call instead of per frame must each be caught. + +Builds go through the front end, verifier, cgen and cbuild in pytest's tmp_path. +""" +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import COMPILED_SHIM +from typedpython import cbuild, cgen, frontend, verify + +SOURCE = """\ +from typing import Callable + + +@compiled +def down(n: int) -> float: + if n == 0: + return 0.0 + return 1.0 + down(n - 1) + + +@compiled +def visit(a: Callable[[int], object], b: Callable[[int], object], n: int) -> None: + a(n) + b(n) + if n > 0: + visit(a, b, n - 1) + + +@compiled +def around(cb: Callable[[], None], m: int) -> float: + cb() + return down(m) + + +@compiled +def seq(g: Callable[[], None], probe: Callable[[], None], m: int) -> float: + g() + probe() + r = down(m) + probe() + return r + + +@compiled +def leaf(i: int) -> float: + return 1.0 / (i + 1) + + +@compiled +def mid(i: int) -> float: + return leaf(i) + leaf(i + 1) + + +@compiled +def deep_mid(n: int) -> float: + if n == 0: + return mid(0) + return 1.0 + deep_mid(n - 1) +""" + +NAMES = {"down", "visit", "around", "seq", "leaf", "mid", "deep_mid"} + + +def generate(d: Path): + path = d / "depth_mod.py" + path.write_text(SOURCE) + proved, _ = verify.verify(frontend.lower(path)) + return path, proved, cgen.generate(proved, path, path.name) + + +def build(d: Path, extra: tuple[str, ...] = (), runtime_dir: Path | None = None): + path, proved, c = generate(d) + so = cbuild.build(c, path.stem, d / "out", runtime_dir=runtime_dir, + flags=cbuild.DEFAULT_FLAGS + extra) + return proved, c, so + + +def mutant_runtime(tmp_path: Path, *edits: tuple[str, str]) -> Path: + mutant = tmp_path / "runtime" + shutil.copytree(cbuild.RUNTIME_DIR, mutant) + text = (mutant / "tp_runtime.h").read_text() + for old, new in edits: + assert text.count(old) == 1, old + text = text.replace(old, new) + (mutant / "tp_runtime.h").write_text(text) + return mutant + + +@pytest.fixture(scope="module") +def generated(tmp_path_factory): + return generate(tmp_path_factory.mktemp("depth_c")) + + +@pytest.fixture(scope="module") +def plain_build(tmp_path_factory): + return build(tmp_path_factory.mktemp("depth_plain")) + + +@pytest.fixture(scope="module") +def plain_src(tmp_path_factory): + p = tmp_path_factory.mktemp("depth_src") / "depth_mod_plain.py" + p.write_text(SOURCE) + return p + + +def fn_text(c: str, signature_start: str) -> str: + """The text of the C function whose definition starts with `signature_start`.""" + i = c.index(signature_start + "(") + while not c[c.index(")", i) + 1:].startswith("\n{"): + i = c.index(signature_start + "(", i + 1) + return c[i:c.index("\n}\n", i)] + + +# --- generated C and the runtime header ---------------------------------------------------------- + +def test_every_function_compiles(generated): + proved = generated[1] + assert {f.name for f in proved.functions} == NAMES, proved.skipped + + +def test_every_wrapper_resets_the_segment_right_around_its_impl_call(generated): + c = generated[2] + for name in NAMES: + w = fn_text(c, f"static PyObject *tp_wrap_{name}").splitlines() + calls = [i for i, line in enumerate(w) if re.match(rf"\s*tp_s = tp_impl_{name}\(", line)] + assert len(calls) == 1, (name, calls) + i = calls[0] + assert w[i - 1].strip() == "tp_depth_seg_begin(tp_t);", (name, w[i - 1]) + assert w[i + 1].strip() == "tp_depth_seg_end(tp_t);", (name, w[i + 1]) + assert sum("tp_depth_seg_" in line for line in w) == 2, name + + +def test_impls_never_reset_or_measure_the_segment_themselves(generated): + c = generated[2] + for name in NAMES: + for sig in (f"static int tp_impl_{name}", f"static inline int tp_fimpl_{name}"): + if sig + "(" not in c: + continue + body = fn_text(c, sig) + assert "tp_depth_seg_" not in body and "tp_python_depth" not in body, sig + assert "PyThreadState_GetFrame" not in body, sig + + +def header_fn(name: str) -> str: + h = (cbuild.RUNTIME_DIR / "tp_runtime.h").read_text() + i = re.search(rf"static inline [^\n(]*\b{name}\(", h).start() + return h[i:h.index("\n}\n", i)] + + +def test_the_frame_checks_use_the_segment_depth_and_keep_the_per_frame_calls(): + enter = header_fn("tp_enter_call") + room = header_fn("tp_depth_room") + for body in (enter, room): + assert "tp_seg_depth(ts, t)" in body and "tp_python_depth" not in body, body + assert "Py_GetRecursionLimit()" in body, body # the limit stays per frame + assert "Py_EnterRecursiveCall(" in enter # the C-stack guard stays per frame + seg = header_fn("tp_seg_depth") + assert "tp_python_depth(ts, t)" in seg and "seg < 0" in seg + + +# --- children: differential runs against the interpreter ----------------------------------------- + +LOAD = COMPILED_SHIM + """\ +import importlib.machinery, importlib.util, json, sys +def load(name, path, ext): + if ext: + loader = importlib.machinery.ExtensionFileLoader(name, path) + spec = importlib.util.spec_from_file_location(name, path, loader=loader) + else: + spec = importlib.util.spec_from_file_location(name, path) + m = importlib.util.module_from_spec(spec) + (loader if ext else spec.loader).exec_module(m) + return m +comp = load("depth_mod", sys.argv[1], True) +plain = load("depth_mod_plain", sys.argv[2], False) +LIMIT = int(sys.argv[3]) +sys.setrecursionlimit(LIMIT) + +def descend(d, fn, a): + if d > 0: + return descend(d - 1, fn, a) + try: + return ["ok", fn(a)] + except RecursionError: + return ["rec"] + +def run(fn, a, d): + try: + return descend(d, fn, a) + except RecursionError: + return ["deep"] + +report = {"mismatch": [], "kinds": {}} +def note(tag, c, p): + if c != p: + report["mismatch"].append([tag, c, p]) + k = report["kinds"].setdefault(tag, {}) + key = c[0] if isinstance(c[0], str) else c[0][0] + k[key] = k.get(key, 0) + 1 + +inner = [] +def mk(mod, k, j): + # a Python callback that descends k Python frames, then makes a nested compiled call down(j) + def cb(): + def dive(i): + if i == 0: + try: + inner.append(["ok", mod.down(j)]) + except RecursionError: + inner.append(["rec"]) + return + dive(i - 1) + dive(k) + return cb + +def nested(mod, d, k, j, m): + del inner[:] + r = run(lambda a: mod.around(mk(mod, k, j), a), m, d) + return [r, list(inner)] +""" + +CHILD_SWEEP = LOAD + """\ +for d in range(1, LIMIT + 6): # every Python depth around the limit + note("down@d", run(comp.down, 20, d), run(plain.down, 20, d)) +for n in range(LIMIT - 40, LIMIT + 6): # every recursion depth around the limit + note("down@n", run(comp.down, n, 5), run(plain.down, n, 5)) + # deep counted recursion, then a bounded callee whose precheck (tp_depth_room) reuses the depth + note("deep_mid", run(comp.deep_mid, n, 5), run(plain.deep_mid, n, 5)) +print(json.dumps(report)) +""" + +CHILD_NESTED = LOAD + """\ +# inner: a nested wrapper inside a segment must measure its own (deeper) Python depth +for j in range(LIMIT - 70, LIMIT + 6): + note("inner", nested(comp, 10, 30, j, 0), nested(plain, 10, 30, j, 0)) +# outer: after the nested wrapper returned, the outer segment must not keep the inner depth +for m in range(LIMIT - 70, LIMIT + 6): + note("outer", nested(comp, 10, 30, 0, m), nested(plain, 10, 30, 0, m)) +print(json.dumps(report)) +""" + +CHILD_LIMIT = LOAD + """\ +class Setter: + def __init__(self, to): + self.to = to + def __del__(self): + sys.setrecursionlimit(self.to) + +def by_del(mod, d, n, at, to): + # visit(a, b, n) calls a(n) then b(n) in every frame; b = objs.pop drops the last element, and + # at n == at that is a Setter whose only reference is the compiled call's result: its __del__ + # runs when the compiled frame releases it (the interpreter: POP_TOP), between two frames + sys.setrecursionlimit(LIMIT) + log = [] + objs = [None] * (n + 1) + objs[at] = Setter(to) + r = run(lambda a: mod.visit(log.append, objs.pop, a), n, d) + sys.setrecursionlimit(LIMIT) + return [r, len(log)] + +def by_callback(mod, d, m, to): + sys.setrecursionlimit(LIMIT) + r = run(lambda a: mod.around(lambda: sys.setrecursionlimit(to), a), m, d) + sys.setrecursionlimit(LIMIT) + return [r] + +N = 2 * LIMIT +for to in range(60, LIMIT - 20, 3): # lowered 20 frames in: the next frames see it + note("del-lower", by_del(comp, 5, N, N - 20, to), by_del(plain, 5, N, N - 20, to)) +for to in (LIMIT + 40, 3 * LIMIT): # raised 20 frames in: the recursion may go deeper + note("del-raise", by_del(comp, 5, N, N - 20, to), by_del(plain, 5, N, N - 20, to)) +for m in range(40, 140, 1): # a callback lowers it before a compiled recursion + note("cb-lower", by_callback(comp, 5, m, 100), by_callback(plain, 5, m, 100)) +print(json.dumps(report)) +""" + +CHILD_FRAMES = LOAD + """\ +events = [] + +class Obj: + def __del__(self): + events.append("del") + +def mkg(mod): + def g(): + x = Obj() # dies with g's frame (or with the frame object that keeps g's locals) + mod.down(1) + return g + +def probe(): + events.append("probe") + +out = {} +del events[:] +comp.seq(mkg(comp), probe, 3) +out["compiled"] = list(events) +del events[:] +plain.seq(mkg(plain), probe, 3) +out["interpreted"] = list(events) + +def chain(): + f, names = sys._getframe(2), [] # from the callback's caller down + while f is not None: + names.append(f.f_code.co_name) + f = f.f_back + return names + +seen = [] +def cb(): + seen.append(chain()) +def cb_nested(): + comp.down(3) # a nested wrapper first: its segment must not change what cb sees + seen.append(chain()) +def cb_nested_plain(): + plain.down(3) + seen.append(chain()) + +def at(d, fn): + if d > 0: + return at(d - 1, fn) + return fn() + +for d in (0, 7, 40): + at(d, lambda: comp.around(cb, 5)) + at(d, lambda: comp.around(cb_nested, 5)) + at(d, lambda: comp.around(cb, 5)) + at(d, lambda: plain.around(cb, 5)) + at(d, lambda: plain.around(cb_nested_plain, 5)) +out["chains"] = seen +print(json.dumps(out)) +""" + +CHILD_THREADS = LOAD + """\ +import threading +results = {} +def worker(t): + mine = [] + for j in range(LIMIT - 60, LIMIT + 6, 2): + mine.append([nested_t(comp, 3 + t, 10 + t, j, 0), nested_t(plain, 3 + t, 10 + t, j, 0)]) + mine.append([nested_t(comp, 3 + t, 10 + t, 0, j), nested_t(plain, 3 + t, 10 + t, 0, j)]) + results[t] = mine + +local = threading.local() +def mk_t(mod, k, j): + def cb(): + def dive(i): + if i == 0: + try: + local.inner.append(["ok", mod.down(j)]) + except RecursionError: + local.inner.append(["rec"]) + return + dive(i - 1) + dive(k) + return cb + +def nested_t(mod, d, k, j, m): + local.inner = [] + r = run(lambda a: mod.around(mk_t(mod, k, j), a), m, d) + return [r, list(local.inner)] + +ts = [threading.Thread(target=worker, args=(t,)) for t in range(4)] +[t.start() for t in ts] +[t.join() for t in ts] +for t, rows in sorted(results.items()): + for c, p in rows: + note(f"thread{t}", c, p) +print(json.dumps(report)) +""" + + +def run_child(script: str, so: Path, plain_src: Path, limit: str = "200") -> dict: + proc = subprocess.run([sys.executable, "-c", script, str(so), str(plain_src), limit], + capture_output=True, text=True, timeout=600) + assert proc.returncode == 0, proc.stdout + proc.stderr + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +def assert_sweep(r): + assert r["mismatch"] == [], r["mismatch"][:5] + for tag in ("down@d", "down@n", "deep_mid"): # the sweep crossed the boundary + assert r["kinds"][tag].get("ok", 0) > 5 and r["kinds"][tag].get("rec", 0) >= 1, (tag, r["kinds"]) + + +def assert_nested(r): + assert r["mismatch"] == [], r["mismatch"][:5] + for tag in ("inner", "outer"): + assert r["kinds"][tag].get("ok", 0) > 5, (tag, r["kinds"]) + # crossing the boundary: some outer runs end in RecursionError, some inner calls do too + assert r["kinds"]["outer"].get("rec", 0) >= 1, r["kinds"] + + +def assert_limit(r): + assert r["mismatch"] == [], r["mismatch"][:5] + assert r["kinds"]["del-lower"].get("rec", 0) >= 10, r["kinds"] # the lowered limit stopped them + assert r["kinds"]["del-raise"].get("ok", 0) >= 1, r["kinds"] # the raised one let one through + k = r["kinds"]["cb-lower"] + assert k.get("ok", 0) > 5 and k.get("rec", 0) > 5, k + + +def test_depth_at_every_python_depth_and_recursion_depth_matches_the_interpreter(plain_build, plain_src): + assert_sweep(run_child(CHILD_SWEEP, plain_build[2], plain_src)) + + +def test_a_nested_compiled_call_from_a_callback_matches_the_interpreter(plain_build, plain_src): + assert_nested(run_child(CHILD_NESTED, plain_build[2], plain_src)) + + +def test_a_limit_changed_by_a_del_or_a_callback_mid_recursion_is_honoured(plain_build, plain_src): + assert_limit(run_child(CHILD_LIMIT, plain_build[2], plain_src, "300")) + + +def test_threads_each_see_their_own_depth(plain_build, plain_src): + r = run_child(CHILD_THREADS, plain_build[2], plain_src) + assert r["mismatch"] == [], r["mismatch"][:5] + assert sorted(r["kinds"]) == [f"thread{t}" for t in range(4)], r["kinds"] + + +def test_frame_lifetimes_and_getframe_are_unchanged(plain_build, plain_src): + """Pins develop's behaviour (green before and after the change). + + Lifetime: inside `seq`, the nested `down(1)` called from the Python callback g measures the depth + with g's frame on top, so the depth cache holds g's frame object; when g returns, that frame + object takes g's locals (`take_ownership`, Python/frame.c), and `x` dies when the outer segment + next measures, at its `down(m)`, not at g's return as in the interpreter. That difference from the + interpreter exists on develop (see the design note); this test only pins that the change keeps + the same moment: a wrapper that kept the inner segment's depth after it returned would never + re-measure and `x` would live until `seq` returns (["probe", "probe", "del"]). + + sys._getframe: the frames a callee sees are the Python frames only (compiled frames have none), + the same at every depth and after a nested compiled call.""" + r = run_child(CHILD_FRAMES, plain_build[2], plain_src) + assert r["interpreted"] == ["del", "probe", "probe"], r + assert r["compiled"] == ["probe", "del", "probe"], r + chains = r["chains"] + assert len(chains) == 15 + for i in range(3): + c1, c2, c3, p1, p2 = chains[5 * i:5 * i + 5] + assert c1 == c2 == c3, (c1, c2, c3) + assert c1 == [n for n in p1 if n not in ("around", "down")], (c1, p1) + assert p1 == p2 + assert len(chains[10]) > len(chains[0]) # deeper at d=40 than at d=0 + + +# --- the measurement really happens once per segment (red before the implementation) ------------- + +def test_one_frame_read_per_segment(tmp_path): + proved, c, so = build(tmp_path, ("-DTP_TRACE_DEPTH",)) + m = cbuild.load("depth_mod", so) + + def reads(fn): + before = m.__tp_trace_frame_reads__() + fn() + return m.__tp_trace_frame_reads__() - before + + # develop measured once per compiled frame: 101, 52 (51 frames + mid's precheck), 1, 101, 18 + assert reads(lambda: m.down(100)) == 1 + assert reads(lambda: m.deep_mid(50)) == 1 + assert reads(lambda: m.mid(3)) == 1 + log: list = [] + objs = [None] * 101 + assert reads(lambda: m.visit(log.append, objs.pop, 100)) == 1 and len(log) == 101 + # around: its own segment, the nested down(5) from the callback, then around measures again + # after that wrapper returned (its down(10) is the first check after the nested segment) + assert reads(lambda: m.around(lambda: m.down(5), 10)) == 3 + + +def test_the_trace_counter_is_absent_from_a_normal_build(plain_build): + c, so = plain_build[1], plain_build[2] + assert "TP_TRACE_DEPTH" in c # guarded, so compiled out + m = cbuild.load("depth_mod", so) + assert not hasattr(m, "__tp_trace_frame_reads__") + + +# --- mutants: each reset and the per-frame limit read are shown to matter ------------------------- + +SEG_BEGIN = "static inline void tp_depth_seg_begin(tp_tls *t) { t->depth.seg = -1; }" +SEG_END = "static inline void tp_depth_seg_end(tp_tls *t) { t->depth.seg = -1; }" +ENTER = "static inline int tp_enter_call(PyThreadState *ts, tp_tls *t)\n{" +ENTER_CHECK = "if (tp_seg_depth(ts, t) + t->depth.count + 1 > Py_GetRecursionLimit()) {" + + +def test_a_wrapper_that_does_not_reset_before_its_impl_is_caught(tmp_path, plain_src): + """Mutation: a nested wrapper reuses the outer segment's (shallower) depth.""" + rt = mutant_runtime(tmp_path, (SEG_BEGIN, "static inline void tp_depth_seg_begin(tp_tls *t) { (void)t; }")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_nested(run_child(CHILD_NESTED, so, plain_src)) + + +def test_a_wrapper_that_does_not_reset_after_its_impl_is_caught(tmp_path, plain_src): + """Mutation: the outer segment keeps the inner segment's (deeper) depth and never re-measures.""" + rt = mutant_runtime(tmp_path, (SEG_END, "static inline void tp_depth_seg_end(tp_tls *t) { (void)t; }")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_nested(run_child(CHILD_NESTED, so, plain_src)) + r = run_child(CHILD_FRAMES, so, plain_src) + assert r["compiled"] == ["probe", "probe", "del"], r # and x lives longer + + +def test_a_limit_read_once_per_call_is_caught(tmp_path, plain_src): + """Mutation: the recursion limit is read at the first compiled frame only (count == 0), as a + once-per-wrapper limit would be. A limit changed mid-recursion is then missed.""" + rt = mutant_runtime( + tmp_path, + (ENTER, "static int tp_mut_limit = 0;\n" + ENTER), + (ENTER_CHECK, "if (t->depth.count == 0) tp_mut_limit = Py_GetRecursionLimit();\n" + " if (tp_seg_depth(ts, t) + t->depth.count + 1 > tp_mut_limit) {")) + (tmp_path / "bad").mkdir() + so = build(tmp_path / "bad", runtime_dir=rt)[2] + with pytest.raises(AssertionError): + assert_limit(run_child(CHILD_LIMIT, so, plain_src, "300")) From d0ce2e9fe2283319838439b7ca7e6c4c00f646fd Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:54:37 +0900 Subject: [PATCH 56/61] =?UTF-8?q?Design=204.3.4=20=EB=B0=94:=20the=20per-c?= =?UTF-8?q?all=20depth=20step=20is=204.3.3b=20(#412)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/design/typedpython.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 1d70a8b2e..f1a8db590 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -524,7 +524,7 @@ CPython 출처는 v3.15.0rc1 태그의 파일과 함수 이름이다. 나갈 때 되돌리면, 프레임마다의 `PyThreadState_GetFrame` 이 없어진다. 첫 측정이 프레임 객체를 이미 만들고 붙잡으므로 새로 생기는 프레임 객체는 없다. 기대 이득: 프로파일의 `PyThreadState_GetFrame` 83 샘플, 약 5% (추정, #273 전 몫). **에이전트 제안, 확인되지 않음.** `tp_tls`, wrapper, 빠른 진입(#147)을 함께 - 바꾸므로 이 단계에서는 하지 않았다. + 바꾸므로 이 단계에서는 하지 않았다. 다음 단계에서 구현: 4.3.3b (#412). - **재귀 한도:** `Py_SetRecursionLimit` 는 3.15t 에서 stop-the-world 로, GIL 빌드에서는 GIL 을 쥐고 바꾼다. 그래서 다른 스레드는 이 스레드가 safepoint 에 닿기 전에는 바꿀 수 없다. 하지만 이 스레드 안에서는 어느 `Py_DECREF` 든 `__del__` 을 부를 수 있고, 거기서 한도를 바꿀 수 있다. 바뀐 것을 알 공개 수단이 없으므로 From 6b29e8ba893cbdfc00fa13980e284f32c3800a56 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:41:50 +0900 Subject: [PATCH 57/61] Chore(compose): bump compose-libraries and composeExtended to 1.11.1-ext.3 (#408) --- .github/workflows/desktop.yml | 2 +- docs/SPEC.md | 6 +++--- gradle.properties | 2 +- gradle/libs.versions.toml | 22 +++++++++------------- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3c741feb9..d4cbd0190 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -230,7 +230,7 @@ jobs: python-multiplatform-compose/fixtures/notebook-e2e/build/test-results/desktopTest # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes the fork's Compose - # (`org.thisisthepy.compose.*:1.11.1-ext.2`, `composeExtended.*` in `gradle.properties`) from + # (`org.thisisthepy.compose.*:1.11.1-ext.3`, `composeExtended.*` in `gradle.properties`) from # Maven Central, builds the 20-scenario executable with Liberica NIK 25, and runs it with # JAVA_HOME and PYTHONHOME removed. notebook-e2e-native-image: diff --git a/docs/SPEC.md b/docs/SPEC.md index a8462a56d..db2e45d37 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -661,14 +661,14 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **B-12** (#408) The Compose fixtures (`python-multiplatform-compose/fixtures/compose`, whose stubs are the `kotlin-stubs` artefact of B-8, and `fixtures/notebook-e2e`) depend on the fork's design systems, `org.thisisthepy.compose.designsystem:designsystem` and `designsystem-{material3, breeze, cupertino, deepin, - fluent, gnome, liquid-glass}` at `compose-design-systems` (`gradle/libs.versions.toml`, 1.11.1-ext.3), and walk + fluent, gnome, liquid-glass}` at `compose-libraries` (`gradle/libs.versions.toml`, `1.11.1-ext.3`), and walk their Kotlin packages `org.thisisthepy.compose.{designsystem, material3, breeze, cupertino, deepin, fluent, gnome, liquidglass}` under those names (decision 12.1). pythonx-compose puts `org.thisisthepy.compose.material3` in front of `androidx.compose.material3` and exposes each design system as `pythonx.compose.` (pythonx-compose #152; leader decision 2026-11-13 on pythonx-compose #54 that they become a dependency here once published). The design systems build against the fork's Compose at their own version, so these two fixtures - resolve the fork's Compose at 1.11.1-ext.3 while the rest of the build stays on `compose-libraries`. - `Status: planned`, blocked on the fork publishing 1.11.1-ext.3 (compose-multiplatform-core-extended #212): + resolve the fork's Compose at 1.11.1-ext.3. + `Status: implemented`: `fixtures/compose/.../DesignSystemBindingTest.kt` and the `org.thisisthepy.compose.material3` lines of `.github/scripts/stubs/consumer.py`. diff --git a/gradle.properties b/gradle.properties index 6ec9af174..43e58a9bb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -47,7 +47,7 @@ org.gradle.java.installations.paths=/Volumes/macMini/toolchains/bellsoft-liberic # compose-multiplatform-core-extended (#191, #249): the Compose modules a native-image build takes from # the fork instead of upstream, and where from. Read in one place, by the notebook E2E's native image; # the fork's coordinates and version line are 1.11.1-ext.N, and only these lines follow. -composeExtended.version=1.11.1-ext.2 +composeExtended.version=1.11.1-ext.3 # The group the fork publishes under, as a prefix of upstream's (org.thisisthepy.compose). composeExtended.groupPrefix=org.thisisthepy.compose composeExtended.modules=org.thisisthepy.compose.ui:ui,org.thisisthepy.compose.ui:ui-desktop,org.thisisthepy.compose.ui:ui-text,org.thisisthepy.compose.ui:ui-text-desktop,org.thisisthepy.compose.foundation:foundation,org.thisisthepy.compose.foundation:foundation-desktop diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 83a08cb87..aa041d549 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,11 +11,7 @@ androidx-espresso-core = "3.6.1" androidx-material = "1.12.0" androidx-test-junit = "1.2.1" compose-plugin = "1.11.1-ext.3" -compose-libraries = "1.11.1-ext.2" -# The fork's design systems (issue #408). They build against the fork's Compose at their own version, -# so a classpath that takes them resolves the fork's Compose at this version too (Gradle picks the -# highest); moving every Compose library to it is a separate change. -compose-design-systems = "1.11.1-ext.3" +compose-libraries = "1.11.1-ext.3" # JetBrains stopped publishing material-icons with Compose after 1.7.3 (Maven Central: newest is 1.7.3); # it is the version Compose Multiplatform 1.11.x points to, resolved against 1.11.1's ui/ui-graphics. compose-material-icons = "1.7.3" @@ -43,14 +39,14 @@ androidx-lifecycle-runtime-compose = { group = "org.thisisthepy.compose.androidx kotlinx-coroutines-swing = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-browser = { group = "org.jetbrains.kotlinx", name = "kotlinx-browser", version.ref = "kotlinx-browser" } # The fork's design systems (compose-multiplatform-core-extended `extended/design-systems`, issue #408). -designsystem = { group = "org.thisisthepy.compose.designsystem", name = "designsystem", version.ref = "compose-design-systems" } -designsystem-material3 = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-material3", version.ref = "compose-design-systems" } -designsystem-breeze = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-breeze", version.ref = "compose-design-systems" } -designsystem-cupertino = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-cupertino", version.ref = "compose-design-systems" } -designsystem-deepin = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-deepin", version.ref = "compose-design-systems" } -designsystem-fluent = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-fluent", version.ref = "compose-design-systems" } -designsystem-gnome = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-gnome", version.ref = "compose-design-systems" } -designsystem-liquid-glass = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-liquid-glass", version.ref = "compose-design-systems" } +designsystem = { group = "org.thisisthepy.compose.designsystem", name = "designsystem", version.ref = "compose-libraries" } +designsystem-material3 = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-material3", version.ref = "compose-libraries" } +designsystem-breeze = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-breeze", version.ref = "compose-libraries" } +designsystem-cupertino = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-cupertino", version.ref = "compose-libraries" } +designsystem-deepin = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-deepin", version.ref = "compose-libraries" } +designsystem-fluent = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-fluent", version.ref = "compose-libraries" } +designsystem-gnome = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-gnome", version.ref = "compose-libraries" } +designsystem-liquid-glass = { group = "org.thisisthepy.compose.designsystem", name = "designsystem-liquid-glass", version.ref = "compose-libraries" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } From 141f26f096ed4b0f92fed28ef98dc6cc283c25b7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:13:44 +0900 Subject: [PATCH 58/61] Docs: guide and README state CPython 3.15 and the 3.15t-only desktop build (#418) The guide still told readers that 3.14t is opt-in through -PpythonFreeThreaded=true; that property was removed with #398 and the build fails when it is passed. --- README.md | 2 +- docs/guide/concepts.html | 2 +- docs/guide/status.html | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a479f7e2c..93c3844b7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ English | [한국어](docs/locale/README_ko.md) [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![Kotlin Multiplatform](https://img.shields.io/badge/Kotlin-Multiplatform-7F52FF.svg?logo=kotlin&logoColor=white)](https://kotlinlang.org/docs/multiplatform.html) -[![CPython](https://img.shields.io/badge/CPython-3.14-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) +[![CPython](https://img.shields.io/badge/CPython-3.15-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) [![Platforms](https://img.shields.io/badge/platforms-JVM%20%7C%20Android%20%7C%20iOS%20%7C%20Android%20native%20%7C%20wasm%20(exp.)-lightgrey.svg)](#-platforms) [Guide](https://thisisthepy.github.io/python-multiplatform/) · diff --git a/docs/guide/concepts.html b/docs/guide/concepts.html index de25f7f0c..deeeba42a 100644 --- a/docs/guide/concepts.html +++ b/docs/guide/concepts.html @@ -74,7 +74,7 @@

03Lifetime across the boundary

CPython counts references; Kotlin collects garbage. A Kotlin wrapper takes one reference and gives it back when the wrapper is collected (a Cleaner, or a PhantomReference path on older Android). Cycles that run through a Kotlin proxy are made visible to Python's cyclic GC, so they are collected too.CPython 은 참조를 세고, Kotlin 은 가비지를 수거한다. Kotlin 래퍼는 참조 하나를 잡고, 래퍼가 수거되면 돌려준다 (Cleaner, 구형 Android 에서는 PhantomReference 경로). Kotlin 프록시를 지나는 순환도 Python 의 순환 GC 에 보이게 하므로 함께 수거된다.

04Threads스레드

-

Every C API call holds the GIL. Parallelism will come from free-threaded CPython (3.14t works on desktop with -PpythonFreeThreaded=true; the default is still the GIL), not from sub-interpreters: extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython(desktop 에서 -PpythonFreeThreaded=true 로 3.14t 동작, 기본값은 여전히 GIL)에서 온다. 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

+

Every C API call holds the GIL. Parallelism comes from free-threaded CPython, not from sub-interpreters (desktop embeds the free-threaded 3.15t and nothing else, with no switch; Android and iOS still embed the GIL build of 3.15 until their 3.15t runtime lands, and so does wasm): extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython 에서 온다(desktop 은 스위치 없이 free-threaded 3.15t 만 내장한다. Android, iOS, wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다). 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

05Names이름

A Kotlin package is a Python module under the same name: androidx.compose.ui in Python is androidx. The binder never renames a namespace. Pythonic layers like pythonx are real Python packages that import those modules and reshape them, they live in their own repository.Kotlin 패키지는 같은 이름의 Python 모듈이다. Python 의 androidx.compose.ui 는 androidx 다. 바인더는 네임스페이스를 바꾸지 않는다. pythonx 같은 파이써닉 계층은 그 모듈을 가져와 다듬는 실제 Python 패키지이고, 자기 저장소에 있다.

diff --git a/docs/guide/status.html b/docs/guide/status.html index d528808c4..6346c7897 100644 --- a/docs/guide/status.html +++ b/docs/guide/status.html @@ -50,7 +50,7 @@

What works today지금 되 Extension functions as methods확장 함수를 메서드로partial부분 .pyi stubs; klib binding.pyi 스텁, klib 바인딩partial부분 Linux and Windows desktop runsLinux, Windows 데스크톱 실행partial부분 -Free-threaded CPython 3.14t (desktop only, opt-in)free-threaded CPython 3.14t (desktop 한정, 옵트인)partial부분 +Free-threaded CPython 3.15t (desktop: the only build; Android, iOS and wasm still embed the GIL build)free-threaded CPython 3.15t (desktop: 유일한 빌드. Android, iOS, wasm 은 아직 GIL 빌드)partial부분 Compose from Python on Android, iOS, wasmAndroid, iOS, wasm 에서 Python 으로 Composeplanned계획

@@ -60,7 +60,7 @@

FAQ

Is it published on Maven Central?Maven Central 에 배포되어 있나?

Not yet. Build from source until a release is announced.아직 아니다. 릴리스가 공지될 때까지 소스에서 빌드한다.

Which Python version?어떤 Python 버전인가?

-

CPython 3.14 today (pythonVersion in gradle.properties). Free-threaded builds can be selected with -PpythonFreeThreaded=true; this works on desktop only (236 tests, 0 failures); Android and iOS have no free-threaded prebuilts.현재 CPython 3.14 다 (gradle.properties 의 pythonVersion). free-threaded 빌드는 -PpythonFreeThreaded=true 로 고를 수 있고, desktop 에서만 동작하며(236 테스트 0 실패) Android/iOS 에는 free-threaded 프리빌트가 없다.

+

CPython 3.15 today (pythonVersion in gradle.properties). Desktop embeds the free-threaded build (3.15t) and has no GIL option; the old -PpythonFreeThreaded switch is removed and the build fails when it is passed. Android, iOS and wasm embed the GIL build of 3.15 until their 3.15t runtimes land.현재 CPython 3.15 다 (gradle.properties 의 pythonVersion). desktop 은 free-threaded 빌드(3.15t)만 내장하고 GIL 옵션이 없다. 예전의 -PpythonFreeThreaded 스위치는 없어졌고 넘기면 빌드가 실패한다. Android, iOS, wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다.

Do C extensions work?C 확장이 동작하나?

Yes, in principle: this is CPython itself, not a re-implementation. Each extension still has to be built for each target platform.원리적으로 그렇다. 재구현이 아니라 CPython 그 자체이기 때문이다. 다만 확장마다 타깃 플랫폼별로 빌드되어 있어야 한다.

Why not reflection?왜 리플렉션을 쓰지 않나?

From 371f713c9fb42f37660e5e524dc8b1fdfd7b85c2 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:06:19 +0900 Subject: [PATCH 59/61] CI: development tests run only by hand, in one ci.yml, one run at a time (#420) The test workflows lose their push triggers and become reusable workflows called by ci.yml, which is dispatched by hand and checks every platform in one run. ci.yml and the CPython builds share the dev-ci concurrency group: a run in progress finishes and a newer dispatch replaces the waiting one. maven-central loses its pull_request dry run; tags still publish. --- .github/workflows/android-native.yml | 13 +---- .github/workflows/build-ft-mobile.yml | 12 ++-- .github/workflows/build-ft-wasm.yml | 20 ++----- .github/workflows/ci.yml | 80 +++++++++++++++++++++++++++ .github/workflows/desktop.yml | 13 +---- .github/workflows/ios.yml | 13 +---- .github/workflows/maven-central.yml | 7 --- .github/workflows/stubs.yml | 9 ++- .github/workflows/typedpython.yml | 16 ++---- .github/workflows/wasm.yml | 13 +---- AGENTS.md | 10 ++++ 11 files changed, 118 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/android-native.yml b/.github/workflows/android-native.yml index 3e77c4f17..c85f06d55 100644 --- a/.github/workflows/android-native.yml +++ b/.github/workflows/android-native.yml @@ -1,16 +1,9 @@ name: Android Native CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/.github/workflows/build-ft-mobile.yml b/.github/workflows/build-ft-mobile.yml index 1cc250d0d..aca3f7525 100644 --- a/.github/workflows/build-ft-mobile.yml +++ b/.github/workflows/build-ft-mobile.yml @@ -6,13 +6,7 @@ name: Build free-threaded CPython for mobile # See docs/platforms/free-threaded-mobile.md. on: - # Runs on develop when the build itself changes (workflow_dispatch needs the file on the - # default branch, main, which develop reaches only through a release). - push: - branches: [develop] - paths: - - ".github/workflows/build-ft-mobile.yml" - - ".github/scripts/ft-mobile/**" + # Started by hand only (owner, 2026-11-14: "개발 테스트 ci는 명시적으로 실행하기"). workflow_dispatch: inputs: python_version: @@ -22,7 +16,9 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # One development run per repository: a run in progress finishes, and a newer dispatch replaces + # the waiting one (owner, 2026-11-14: "제일 최신꺼만 도는게 맞지"). + group: dev-ci cancel-in-progress: false jobs: diff --git a/.github/workflows/build-ft-wasm.yml b/.github/workflows/build-ft-wasm.yml index 6baebdbe6..25f2f6c58 100644 --- a/.github/workflows/build-ft-wasm.yml +++ b/.github/workflows/build-ft-wasm.yml @@ -13,21 +13,7 @@ name: Build free-threaded CPython for wasm # shared memory (trampoline_patch=skip runs without it). on: - # Runs on develop when the build itself changes (workflow_dispatch needs the file on the - # default branch, main, which develop reaches only through a release). - push: - branches: [develop] - paths: - - ".github/workflows/build-ft-wasm.yml" - - ".github/scripts/ft-wasm/**" - - ".github/scripts/ft-mobile/common.sh" - # And on a pull request that changes it, so a change is built before it merges (a new workflow - # cannot be dispatched until it reaches main). - pull_request: - paths: - - ".github/workflows/build-ft-wasm.yml" - - ".github/scripts/ft-wasm/**" - - ".github/scripts/ft-mobile/common.sh" + # Started by hand only (owner, 2026-11-14: "개발 테스트 ci는 명시적으로 실행하기"). workflow_dispatch: inputs: python_version: @@ -48,7 +34,9 @@ on: options: [apply, skip] concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # One development run per repository: a run in progress finishes, and a newer dispatch replaces + # the waiting one (owner, 2026-11-14: "제일 최신꺼만 도는게 맞지"). + group: dev-ci cancel-in-progress: false jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..518fccb45 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: CI + +# The one development CI of this repository. It is started by hand only (owner, 2026-11-14: +# "개발 테스트 ci는 명시적으로 실행하기", "한 ci에서 최대한 여러개를 확인하기"), and one run checks +# every platform: each suite below is a reusable workflow, on by default. +# +# gh workflow run ci.yml --ref +# +# The concurrency group is shared with the CPython builds (build-ft-*.yml): a run in progress +# finishes, a newer dispatch replaces the waiting one, and nothing else runs beside it (owner: +# "제일 최신꺼만 도는게 맞지", "여러 ci를 한번에 돌리는거도 허용할 생각이 난 없어"). + +on: + workflow_dispatch: + inputs: + desktop: + description: "Desktop (JVM, macosArm64, linuxX64) and the generator fixtures" + type: boolean + default: true + android: + description: "Android and androidNative" + type: boolean + default: true + ios: + description: "iOS" + type: boolean + default: true + wasm: + description: "WasmJS" + type: boolean + default: true + typedpython: + description: "TypedPython gate and compiler" + type: boolean + default: true + race_mutant: + description: "TypedPython: also run the race_mutant tests" + type: boolean + default: false + stubs: + description: "Kotlin-name .pyi stubs" + type: boolean + default: true + +concurrency: + group: dev-ci + cancel-in-progress: false + +permissions: + contents: read + +jobs: + desktop: + if: ${{ inputs.desktop }} + uses: ./.github/workflows/desktop.yml + secrets: inherit + android: + if: ${{ inputs.android }} + uses: ./.github/workflows/android-native.yml + secrets: inherit + ios: + if: ${{ inputs.ios }} + uses: ./.github/workflows/ios.yml + secrets: inherit + wasm: + if: ${{ inputs.wasm }} + uses: ./.github/workflows/wasm.yml + secrets: inherit + typedpython: + if: ${{ inputs.typedpython }} + uses: ./.github/workflows/typedpython.yml + with: + race_mutant: ${{ inputs.race_mutant }} + secrets: inherit + stubs: + if: ${{ inputs.stubs }} + uses: ./.github/workflows/stubs.yml + permissions: + contents: write + secrets: inherit diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3c741feb9..fab85eef2 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -1,16 +1,9 @@ name: Desktop CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: test: diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index f1287532b..ee7ee9fb6 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -1,16 +1,9 @@ name: iOS CI on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml index d22eca2ee..10d22994d 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/maven-central.yml @@ -21,13 +21,6 @@ on: push: tags: - 'v*' - # A pull request that touches the publishing setup runs the dry run, so the bundle is checked - # before the first tag depends on it. - pull_request: - paths: - - '.github/workflows/maven-central.yml' - - '.github/scripts/release/central-*.sh' - - 'gradle/central-publishing.gradle.kts' workflow_dispatch: inputs: dry_run: diff --git a/.github/workflows/stubs.yml b/.github/workflows/stubs.yml index ed99ecd36..9cfb2879f 100644 --- a/.github/workflows/stubs.yml +++ b/.github/workflows/stubs.yml @@ -4,17 +4,16 @@ name: Kotlin-name stubs # but cannot run this repository's Gradle build. This job runs the stub task over the Compose version the # build resolves (`compose-libraries` in gradle/libs.versions.toml) and publishes the result. # -# - every push to develop and every pull request: the stubs are generated, type-checked against +# - every ci.yml run: the stubs are generated, type-checked against # .github/scripts/stubs/consumer.py with mypy (issue #31), and uploaded as the workflow artifact `kotlin-stubs`; # - every `v*` tag: the same stubs are also attached to the tag's GitHub release as `kotlin-stubs.zip`. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. + # Development runs come only through ci.yml, which is started by hand (owner, 2026-11-14: + # "개발 테스트 ci는 명시적으로 실행하기"). A release tag still attaches the stubs. push: - branches: [ "develop" ] tags: [ "v*" ] - workflow_dispatch: + workflow_call: permissions: contents: read diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml index 70c97fee9..b8b266d9b 100644 --- a/.github/workflows/typedpython.yml +++ b/.github/workflows/typedpython.yml @@ -3,12 +3,9 @@ name: TypedPython CI # The TypedPython gate and compiler are Python (python-multiplatform-ksp/src/{main,test}/python). # Gradle does not run them; this workflow does. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - paths: [ "python-multiplatform-ksp/**", "python-multiplatform-ksp/fixtures/app/**", ".github/workflows/typedpython.yml" ] - workflow_dispatch: + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: inputs: race_mutant: # The race_mutant tests show that a concurrency test catches a planted mutant; each depends @@ -16,11 +13,6 @@ on: description: "Also run the race_mutant tests (a missed race fails the step)" type: boolean default: false -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false jobs: test: @@ -58,7 +50,7 @@ jobs: run: ${{ matrix.venv_python }} -m pytest -q -m "not measurement and not race_mutant" - name: Race mutants (dispatch input race_mutant) - if: ${{ github.event_name == 'workflow_dispatch' && inputs.race_mutant }} + if: ${{ inputs.race_mutant }} run: ${{ matrix.venv_python }} -m pytest -q -rA -m race_mutant - name: Measurements (recorded, sanity bounds only) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 43320e8cd..bb0149b7b 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -13,16 +13,9 @@ name: WasmJS CI # `test` runs the real suite, and runs only when there is a runtime to run it against. on: - # Checked on develop after merging, not on every pull request (user decision, - # 2026-10-17). A red develop is fixed forward by its own pull request. - push: - branches: [ "develop", "main" ] - workflow_dispatch: -concurrency: - # A run in progress finishes; a newer push replaces only the queued run (frequent merges - # to develop otherwise cancelled every run before it reported). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false + # Runs only as part of ci.yml, which is started by hand (owner, 2026-11-14: "개발 테스트 ci는 + # 명시적으로 실행하기"). ci.yml holds the one concurrency group for the repository. + workflow_call: jobs: compile: diff --git a/AGENTS.md b/AGENTS.md index d06dfc06d..54c8c0cf0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -194,6 +194,16 @@ to implement", say what you counted against. branch. The only local exceptions are device tests and measurements the coordinating session runs itself, one at a time, and says why. Several sessions share one machine, and agent builds once filled the disk and slowed every session. +- **CI runs only when started by hand, one run at a time.** Owner, 2026-11-14: "여러 ci를 + 쌓아두는거도 말이 안되잖아. 제일 최신꺼만 도는게 맞지. 애초에 여러 ci를 한번에 돌리는거도 허용할 + 생각이 난 없어." Development test workflows have no `push`, `pull_request` or `schedule` trigger. + The coordinating session starts the repository's one development workflow with + `gh workflow run --ref ` on the latest commit; agents and teammates never start, + re-run or queue CI themselves, they ask the coordinator. One run checks as much as it can: every + suite in one dispatch, and several pull requests merged onto one branch rather than one run each. + A run in progress finishes and a newer dispatch replaces the waiting one (one `dev-ci` + concurrency group per repository). Release workflows (tags, published releases, release-sync, + pages) keep their triggers. --- From d83d11efd251607adf834805c48916e40cb7fbd7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:05:10 +0900 Subject: [PATCH 60/61] Feat(compose): bump compose-libraries to 1.11.1-ext.4 and fold companion-constant receiver classes (#408) --- .github/workflows/desktop.yml | 2 +- docs/SPEC.md | 4 +- gradle.properties | 2 +- gradle/libs.versions.toml | 2 +- .../gradle/artifact/ArtifactScanner.kt | 26 ++++- .../gradle/stubs/PyiRendering.kt | 42 ++++++-- .../gradle/stubs/CompanionHandleStubTest.kt | 97 +++++++++++++++++++ .../ffi/pythonx/PythonxAdapter.kt | 2 +- 8 files changed, 157 insertions(+), 20 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 80d3c80b3..fab85eef2 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -223,7 +223,7 @@ jobs: python-multiplatform-compose/fixtures/notebook-e2e/build/test-results/desktopTest # The notebook E2E in a GraalVM native image on macOS (#249, #251). It takes the fork's Compose - # (`org.thisisthepy.compose.*:1.11.1-ext.3`, `composeExtended.*` in `gradle.properties`) from + # (`org.thisisthepy.compose.*:1.11.1-ext.2`, `composeExtended.*` in `gradle.properties`) from # Maven Central, builds the 20-scenario executable with Liberica NIK 25, and runs it with # JAVA_HOME and PYTHONHOME removed. notebook-e2e-native-image: diff --git a/docs/SPEC.md b/docs/SPEC.md index a5a8a4b6e..768630dad 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -661,13 +661,13 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **B-12** (#408) The Compose fixtures (`python-multiplatform-compose/fixtures/compose`, whose stubs are the `kotlin-stubs` artefact of B-8, and `fixtures/notebook-e2e`) depend on the fork's design systems, `org.thisisthepy.compose.designsystem:designsystem` and `designsystem-{material3, breeze, cupertino, deepin, - fluent, gnome, liquid-glass}` at `compose-libraries` (`gradle/libs.versions.toml`, `1.11.1-ext.3`), and walk + fluent, gnome, liquid-glass}` at `compose-libraries` (`gradle/libs.versions.toml`, `1.11.1-ext.4`), and walk their Kotlin packages `org.thisisthepy.compose.{designsystem, material3, breeze, cupertino, deepin, fluent, gnome, liquidglass}` under those names (decision 12.1). pythonx-compose puts `org.thisisthepy.compose.material3` in front of `androidx.compose.material3` and exposes each design system as `pythonx.compose.` (pythonx-compose #152; leader decision 2026-11-13 on pythonx-compose #54 that they become a dependency here once published). The design systems build against the fork's Compose at their own version, so these two fixtures - resolve the fork's Compose at 1.11.1-ext.3. + resolve the fork's Compose at 1.11.1-ext.4. `Status: implemented`: `fixtures/compose/.../DesignSystemBindingTest.kt` and the `org.thisisthepy.compose.material3` lines of `.github/scripts/stubs/consumer.py`. diff --git a/gradle.properties b/gradle.properties index 3bae2c4e2..4dc862800 100644 --- a/gradle.properties +++ b/gradle.properties @@ -46,7 +46,7 @@ org.gradle.java.installations.paths=/Volumes/macMini/toolchains/bellsoft-liberic # compose-multiplatform-core-extended (#191, #249): the Compose modules a native-image build takes from # the fork instead of upstream, and where from. Read in one place, by the notebook E2E's native image; # the fork's coordinates and version line are 1.11.1-ext.N, and only these lines follow. -composeExtended.version=1.11.1-ext.3 +composeExtended.version=1.11.1-ext.4 # The group the fork publishes under, as a prefix of upstream's (org.thisisthepy.compose). composeExtended.groupPrefix=org.thisisthepy.compose composeExtended.modules=org.thisisthepy.compose.ui:ui,org.thisisthepy.compose.ui:ui-desktop,org.thisisthepy.compose.ui:ui-text,org.thisisthepy.compose.ui:ui-text-desktop,org.thisisthepy.compose.foundation:foundation,org.thisisthepy.compose.foundation:foundation-desktop diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index aa041d549..27581ebd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,7 +11,7 @@ androidx-espresso-core = "3.6.1" androidx-material = "1.12.0" androidx-test-junit = "1.2.1" compose-plugin = "1.11.1-ext.3" -compose-libraries = "1.11.1-ext.3" +compose-libraries = "1.11.1-ext.4" # JetBrains stopped publishing material-icons with Compose after 1.7.3 (Maven Central: newest is 1.7.3); # it is the version Compose Multiplatform 1.11.x points to, resolved against 1.11.1's ui/ui-graphics. compose-material-icons = "1.7.3" diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt index 70babf976..dabd6980d 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt @@ -462,7 +462,8 @@ internal object ArtifactScanner { functions = functionsOf(metadata.kmPackage), ownerNode = node, classpath = classpath, - ) + extensionPropertyCandidates(metadata.kmPackage, node, owner, classpath) + ) + extensionPropertyCandidates(metadata.kmPackage, node, owner, classpath) + + constantsOf(metadata.kmPackage.properties, node, boundOwner = owner, callOwner = owner, classpath = classpath, ownerIsClass = false) } is KotlinClassMetadata.MultiFileClassFacade -> metadata.partClassNames.flatMap { partBinaryName -> val partNode = classpath.classNode(partBinaryName) ?: return@flatMap emptyList() @@ -475,7 +476,8 @@ internal object ArtifactScanner { functions = functionsOf(partMetadata.kmPackage), ownerNode = partNode, classpath = classpath, - ) + extensionPropertyCandidates(partMetadata.kmPackage, partNode, owner, classpath) + ) + extensionPropertyCandidates(partMetadata.kmPackage, partNode, owner, classpath) + + constantsOf(partMetadata.kmPackage.properties, partNode, boundOwner = owner, callOwner = owner, classpath = classpath, ownerIsClass = false) } // A lone part is package-private and never reaches here on its own (the `ACC_PUBLIC` // guard above already excluded it) -- only through the facade case, which fetches it by @@ -875,7 +877,23 @@ internal object ArtifactScanner { boundOwner: String, callOwner: String, classpath: ArtifactClasspath, - ): List = kmClass.properties.mapNotNull { property -> + ): List = constantsOf( + properties = kmClass.properties, + declaringNode = declaringNode, + boundOwner = boundOwner, + callOwner = callOwner, + classpath = classpath, + ownerIsClass = true, + ) + + private fun constantsOf( + properties: List, + declaringNode: ClassNode, + boundOwner: String, + callOwner: String, + classpath: ArtifactClasspath, + ownerIsClass: Boolean = true, + ): List = properties.mapNotNull { property -> if (property.visibility != Visibility.PUBLIC) return@mapNotNull null if (property.receiverParameterType != null) return@mapNotNull null // A `@Composable get()` reads the composition, so its value does not exist outside one -- @@ -928,7 +946,7 @@ internal object ArtifactScanner { declaration = DeclarationModel( simpleName = property.name, owner = boundOwner, - ownerIsClass = true, + ownerIsClass = ownerIsClass, receiver = null, parameters = emptyList(), returnType = returnModel, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt index 0460c5db3..96ef89ddf 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/PyiRendering.kt @@ -280,22 +280,49 @@ private class StubRenderer(declarations: List) { /** Class qualified name -> its bound constructors, in table-key order; filled by [renderModuleDefs]. */ private val constructorsByClass = sortedMapOf>() + private val supertypes: Map> = bound + .filter { it.returnSupertypes.isNotEmpty() } + .groupBy { it.returnType.qualifiedName } + .mapValues { (_, entries) -> entries.first().returnSupertypes } + /** - * Module prefixes (every module and every directory above it) that name one directory on a - * case-insensitive filesystem together with a sibling (issue #44). In each such group the - * all-lower-case spelling keeps its path -- a Kotlin package -- and every other spelling is a - * class module that is emitted inside its parent package's `__init__.pyi` instead. + * Module prefixes folded into their parent package's `__init__.pyi`. + * + * Two reasons fold a class module into its parent: + * (a) Case-insensitive path collision with a sibling (issue #44): in each such group the + * all-lower-case spelling keeps its path -- a Kotlin package -- and every other spelling + * is a class module emitted inside its parent package's `__init__.pyi`. + * (b) A receiver class (with instance methods or properties and a companion handle) whose companion + * module only publishes constants (`Material3DesignSystem.Dark`), where the runtime + * (`class_wins_over_module`, issue #322) serves the class on the parent package and the constants + * on the class, provided `module` is not a supertype of another bound type (so an interface like + * `DrawScope` stays a module while a concrete receiver class like `Material3DesignSystem` is folded). */ private val foldedPrefixes: Set = run { val prefixes = byModule.keys.flatMap { module -> val parts = module.split('.') parts.indices.map { parts.take(it + 1).joinToString(".") } }.toSet() - prefixes.groupBy { it.lowercase() }.values + val caseColliding = prefixes.groupBy { it.lowercase() }.values .filter { it.size > 1 } .flatten() .filter { '.' in it && it.substringAfterLast('.') != it.substringAfterLast('.').lowercase() } .toSet() + val allSupertypes = supertypes.values.flatten().toSet() + val companionConstantClasses = byModule.keys.filter { module -> + module !in allSupertypes && + module in companionHandles && + (module in methods || module in properties) && + byModule[module]?.all { it.kind == "STATIC_GETTER" } == true && + module.substringAfterLast('.') !in takenNames[module.substringBeforeLast('.', "")].orEmpty() && + run { + val prefix = "$module." + bound.none { it.returnType.qualifiedName.startsWith(prefix) || it.owner.startsWith(prefix) } && + methods.keys.none { it.startsWith(prefix) } && + properties.keys.none { it.startsWith(prefix) } + } + }.toSet() + caseColliding + companionConstantClasses } /** The shortest folded prefix of [module] (the module itself included), or `null` if it keeps its own path. */ @@ -307,11 +334,6 @@ private class StubRenderer(declarations: List) { /** The modules that are still files: a folded class module is a class, not a module, for `classReference`. */ private val moduleNames: Set = byModule.keys.filter { foldedPrefixOf(it) == null }.toSet() - private val supertypes: Map> = bound - .filter { it.returnSupertypes.isNotEmpty() } - .groupBy { it.returnType.qualifiedName } - .mapValues { (_, entries) -> entries.first().returnSupertypes } - // ----------------------------------------------------------------------- per-module output private class ClassNode(val name: String) { diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt index a2b11a193..82cca5f33 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/stubs/CompanionHandleStubTest.kt @@ -1,6 +1,7 @@ package python.multiplatform.gradle.stubs import python.multiplatform.gradle.model.DeclarationModel +import python.multiplatform.gradle.model.DeclaredParameter import python.multiplatform.gradle.model.KotlinTypeModel import kotlin.test.Test import kotlin.test.assertFalse @@ -60,4 +61,100 @@ class CompanionHandleStubTest { val body = stubs.getValue("androidx/compose/ui/__init__.pyi") assertFalse("receiver: _t.Any" in body, body) } + + @Test + fun classWithCompanionHandleInstanceMembersAndCompanionConstantsIsFoldedIntoClassStub() { + val m3Pkg = "androidx.compose.material3" + val designSystem = "$m3Pkg.Material3DesignSystem" + val designSystemType = KotlinTypeModel(designSystem) + val colorSchemeType = KotlinTypeModel("$m3Pkg.ColorScheme") + + val companionHandle = DeclarationModel( + simpleName = "Companion", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Companion", + isCompanionHandle = true, + ) + + val dark = DeclarationModel( + simpleName = "Dark", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Dark", + ) + + val light = DeclarationModel( + simpleName = "Light", + owner = designSystem, + ownerIsClass = true, + receiver = null, + parameters = emptyList(), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "STATIC_GETTER", + bindingName = "$designSystem.Light", + ) + + val method = DeclarationModel( + simpleName = "validate", + owner = designSystem, + ownerIsClass = true, + receiver = designSystemType, + receiverBoundaryTag = "OBJECT", + parameters = emptyList(), + returnType = KotlinTypeModel("kotlin.Unit"), + returnBoundaryTag = "UNIT", + kind = "METHOD", + bindingName = "$designSystem.validate", + ) + + val property = DeclarationModel( + simpleName = "colorScheme", + owner = designSystem, + ownerIsClass = true, + receiver = designSystemType, + receiverBoundaryTag = "OBJECT", + parameters = emptyList(), + returnType = colorSchemeType, + returnBoundaryTag = "OBJECT", + kind = "GETTER", + bindingName = "$designSystem.colorScheme", + ) + + val usage = DeclarationModel( + simpleName = "useSystem", + owner = m3Pkg, + ownerIsClass = false, + receiver = null, + parameters = listOf(DeclaredParameter("system", designSystemType, declaresDefault = false, boundaryTag = "OBJECT")), + returnType = designSystemType, + returnBoundaryTag = "OBJECT", + kind = "FUNCTION", + bindingName = "$m3Pkg.useSystem", + ) + + val files = renderKotlinFqnStubs(listOf(companionHandle, dark, light, method, property, usage)) + assertFalse(files.keys.any { it.startsWith("androidx/compose/material3/Material3DesignSystem") }, files.keys.toString()) + + val body = files.getValue("androidx/compose/material3/__init__.pyi") + assertTrue("class Material3DesignSystem:" in body, body) + assertTrue(" Dark: Material3DesignSystem" in body, body) + assertTrue(" Light: Material3DesignSystem" in body, body) + assertTrue(" def validate(self) -> None:" in body, body) + assertTrue(" @property\n def colorScheme(self) -> ColorScheme:" in body, body) + assertTrue("def useSystem(system: Material3DesignSystem) -> Material3DesignSystem:" in body, body) + assertFalse("system: _t.Any" in body, body) + assertFalse("-> _t.Any" in body, body) + } } diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt index 25aa786c9..2281a5327 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt @@ -2293,7 +2293,7 @@ object PythonxAdapter { def __dir__(): names = set(_BY_PACKAGE.get(kotlin_package, {})) names.update(_CHILDREN.get(kotlin_package, ())) - for served in (_BY_RECEIVER, _METHODS): + for served in (_BY_RECEIVER, _METHODS, _PROPERTIES): for type_name in served: package, _, leaf = type_name.rpartition('.') if package == kotlin_package: From d9359bc04765fe96b33554cbf5291153bec5f552 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:12:06 +0900 Subject: [PATCH 61/61] Docs: guide and README_ko state Android and iOS also embed 3.15t (#418) --- docs/guide/concepts.html | 2 +- docs/guide/status.html | 4 ++-- docs/locale/README_ko.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/guide/concepts.html b/docs/guide/concepts.html index deeeba42a..48bf7fe17 100644 --- a/docs/guide/concepts.html +++ b/docs/guide/concepts.html @@ -74,7 +74,7 @@

03Lifetime across the boundary

CPython counts references; Kotlin collects garbage. A Kotlin wrapper takes one reference and gives it back when the wrapper is collected (a Cleaner, or a PhantomReference path on older Android). Cycles that run through a Kotlin proxy are made visible to Python's cyclic GC, so they are collected too.CPython 은 참조를 세고, Kotlin 은 가비지를 수거한다. Kotlin 래퍼는 참조 하나를 잡고, 래퍼가 수거되면 돌려준다 (Cleaner, 구형 Android 에서는 PhantomReference 경로). Kotlin 프록시를 지나는 순환도 Python 의 순환 GC 에 보이게 하므로 함께 수거된다.

04Threads스레드

-

Every C API call holds the GIL. Parallelism comes from free-threaded CPython, not from sub-interpreters (desktop embeds the free-threaded 3.15t and nothing else, with no switch; Android and iOS still embed the GIL build of 3.15 until their 3.15t runtime lands, and so does wasm): extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython 에서 온다(desktop 은 스위치 없이 free-threaded 3.15t 만 내장한다. Android, iOS, wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다). 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

+

Every C API call holds the GIL. Parallelism comes from free-threaded CPython, not from sub-interpreters (desktop, Android and iOS embed the free-threaded 3.15t and nothing else, with no switch; wasm still embeds the GIL build of 3.15 until its 3.15t runtime lands): extensions that do not declare multi-interpreter support fail to import in a sub-interpreter.모든 C API 호출은 GIL 을 잡는다. 병렬성은 서브 인터프리터가 아니라 free-threaded CPython 에서 온다(desktop, Android, iOS 는 스위치 없이 free-threaded 3.15t 만 내장한다. wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다). 멀티 인터프리터 지원을 선언하지 않은 확장은 서브 인터프리터에서 import 가 실패하기 때문이다.

05Names이름

A Kotlin package is a Python module under the same name: androidx.compose.ui in Python is androidx. The binder never renames a namespace. Pythonic layers like pythonx are real Python packages that import those modules and reshape them, they live in their own repository.Kotlin 패키지는 같은 이름의 Python 모듈이다. Python 의 androidx.compose.ui 는 androidx 다. 바인더는 네임스페이스를 바꾸지 않는다. pythonx 같은 파이써닉 계층은 그 모듈을 가져와 다듬는 실제 Python 패키지이고, 자기 저장소에 있다.

diff --git a/docs/guide/status.html b/docs/guide/status.html index 6346c7897..414ec4393 100644 --- a/docs/guide/status.html +++ b/docs/guide/status.html @@ -50,7 +50,7 @@

What works today지금 되 Extension functions as methods확장 함수를 메서드로partial부분 .pyi stubs; klib binding.pyi 스텁, klib 바인딩partial부분 Linux and Windows desktop runsLinux, Windows 데스크톱 실행partial부분 -Free-threaded CPython 3.15t (desktop: the only build; Android, iOS and wasm still embed the GIL build)free-threaded CPython 3.15t (desktop: 유일한 빌드. Android, iOS, wasm 은 아직 GIL 빌드)partial부분 +Free-threaded CPython 3.15t (desktop, Android and iOS: the only build; wasm still embeds the GIL build)free-threaded CPython 3.15t (desktop, Android, iOS: 유일한 빌드. wasm 은 아직 GIL 빌드)partial부분 Compose from Python on Android, iOS, wasmAndroid, iOS, wasm 에서 Python 으로 Composeplanned계획 @@ -60,7 +60,7 @@

FAQ

Is it published on Maven Central?Maven Central 에 배포되어 있나?

Not yet. Build from source until a release is announced.아직 아니다. 릴리스가 공지될 때까지 소스에서 빌드한다.

Which Python version?어떤 Python 버전인가?

-

CPython 3.15 today (pythonVersion in gradle.properties). Desktop embeds the free-threaded build (3.15t) and has no GIL option; the old -PpythonFreeThreaded switch is removed and the build fails when it is passed. Android, iOS and wasm embed the GIL build of 3.15 until their 3.15t runtimes land.현재 CPython 3.15 다 (gradle.properties 의 pythonVersion). desktop 은 free-threaded 빌드(3.15t)만 내장하고 GIL 옵션이 없다. 예전의 -PpythonFreeThreaded 스위치는 없어졌고 넘기면 빌드가 실패한다. Android, iOS, wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다.

+

CPython 3.15 today (pythonVersion in gradle.properties). Desktop, Android and iOS embed the free-threaded build (3.15t) and have no GIL option; the old -PpythonFreeThreaded switch is removed and the build fails when it is passed. Wasm embeds the GIL build of 3.15 until its 3.15t runtime lands.현재 CPython 3.15 다 (gradle.properties 의 pythonVersion). desktop, Android, iOS 는 free-threaded 빌드(3.15t)만 내장하고 GIL 옵션이 없다. 예전의 -PpythonFreeThreaded 스위치는 없어졌고 넘기면 빌드가 실패한다. wasm 은 3.15t 런타임이 들어올 때까지 GIL 빌드 3.15 를 내장한다.

Do C extensions work?C 확장이 동작하나?

Yes, in principle: this is CPython itself, not a re-implementation. Each extension still has to be built for each target platform.원리적으로 그렇다. 재구현이 아니라 CPython 그 자체이기 때문이다. 다만 확장마다 타깃 플랫폼별로 빌드되어 있어야 한다.

Why not reflection?왜 리플렉션을 쓰지 않나?

diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md index 8d63c90b2..65a5e3d9d 100644 --- a/docs/locale/README_ko.md +++ b/docs/locale/README_ko.md @@ -8,7 +8,7 @@ [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](../../LICENSE) [![Kotlin Multiplatform](https://img.shields.io/badge/Kotlin-Multiplatform-7F52FF.svg?logo=kotlin&logoColor=white)](https://kotlinlang.org/docs/multiplatform.html) -[![CPython](https://img.shields.io/badge/CPython-3.14-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) +[![CPython](https://img.shields.io/badge/CPython-3.15-3776AB.svg?logo=python&logoColor=white)](https://www.python.org/) [![Platforms](https://img.shields.io/badge/platforms-JVM%20%7C%20Android%20%7C%20iOS%20%7C%20Android%20native%20%7C%20wasm%20(exp.)-lightgrey.svg)](#-플랫폼) [가이드](https://thisisthepy.github.io/python-multiplatform/) ·