From c9777442d9ec9377fddae0a2c02d15a31b268f21 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 14:50:42 +0200 Subject: [PATCH 1/6] prototype: ship prebuilt abi3 rtree modules, keep witty as fallback --- hatch_build.py | 156 +++++++++++++++++++++++++++ pyproject.toml | 12 ++- src/spatial_graph/__init__.py | 34 +++++- src/spatial_graph/_rtree/_codegen.py | 41 +++++++ src/spatial_graph/_rtree/_naming.py | 49 +++++++++ src/spatial_graph/_rtree/_specs.py | 40 +++++++ src/spatial_graph/_rtree/rtree.py | 63 ++++++----- tests/test_prebuilt.py | 75 +++++++++++++ 8 files changed, 439 insertions(+), 31 deletions(-) create mode 100644 hatch_build.py create mode 100644 src/spatial_graph/_rtree/_codegen.py create mode 100644 src/spatial_graph/_rtree/_naming.py create mode 100644 src/spatial_graph/_rtree/_specs.py create mode 100644 tests/test_prebuilt.py diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..af48e78 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,156 @@ +"""Compile RTree variants ahead of time into stable-ABI (abi3) wheels. + +Renders the same pyx wrappers the runtime would JIT-compile (via +`_rtree._codegen`) for every variant in `_rtree._specs`, builds them against +`Py_LIMITED_API`, and force-includes the result as +`spatial_graph/_rtree/_prebuilt/`. One wheel per platform then covers every +supported CPython, and users never need a C compiler for those variants. + +Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead. +""" + +from __future__ import annotations + +import os +import sys +import sysconfig +from pathlib import Path +from typing import Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# oldest CPython with the buffer protocol (memoryviews) in the limited API +ABI3_MIN = (3, 11) +ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" + +ROOT = Path(__file__).parent +SRC = ROOT / "src" +PKG = "spatial_graph/_rtree/_prebuilt" + + +def _platform_tag() -> str: + return sysconfig.get_platform().replace("-", "_").replace(".", "_") + + +def _stub_spatial_graph_package() -> None: + """Make `spatial_graph.*` submodules importable without running its `__init__`. + + `spatial_graph/__init__.py` pulls in the graph half, and with it witty; the + build only needs the rtree codegen, so we register a bare namespace package + pointing at the source tree instead. + """ + import types + + pkg = types.ModuleType("spatial_graph") + pkg.__path__ = [str(SRC / "spatial_graph")] # type: ignore[attr-defined] + sys.modules.setdefault("spatial_graph", pkg) + + +class PrebuiltRTreeHook(BuildHookInterface): + PLUGIN_NAME = "prebuilt-rtree" + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + if self.target_name != "wheel": + return + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return + + # 3.10 lacks the buffer protocol in the limited API, so it gets a plain + # version-specific wheel; 3.11+ all share one abi3 wheel per platform. + abi3 = sys.version_info >= ABI3_MIN + + _stub_spatial_graph_package() + from spatial_graph._rtree._codegen import build_wrapper + from spatial_graph._rtree._naming import module_name + from spatial_graph._rtree._specs import iter_specs + + build_dir = ROOT / "build" / "prebuilt" / f"{_platform_tag()}-{abi3}" + pyx_dir = build_dir / "pyx" + pyx_dir.mkdir(parents=True, exist_ok=True) + + names = [] + for spec in iter_specs(): + name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + source = build_wrapper( + spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims + ) + path = pyx_dir / f"{name}.pyx" + # only rewrite when changed, so cythonize can skip unchanged variants + if not path.is_file() or path.read_text() != source: + path.write_text(source) + names.append(name) + + try: + built = self._compile(pyx_dir, names, build_dir, abi3) + except Exception as e: + # Installing from an sdist on a machine with no usable compiler must + # keep working: fall back to a pure-Python wheel that JIT-compiles on + # first use, exactly as before prebuilding existed. CI sets + # SPATIAL_GRAPH_REQUIRE_PREBUILT so this can never pass silently there. + if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + raise + self.app.display_warning( + f"Could not prebuild rtree modules ({e}); building a pure-Python " + "wheel. A C compiler will be needed the first time an RTree is used." + ) + return + + force_include = build_data.setdefault("force_include", {}) + init = build_dir / "__init__.py" + init.write_text("") + force_include[str(init)] = f"{PKG}/__init__.py" + for artifact in built: + force_include[str(artifact)] = f"{PKG}/{artifact.name}" + + build_data["pure_python"] = False + if abi3: + build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" + else: + build_data["infer_tag"] = True + + def _compile( + self, pyx_dir: Path, names: list[str], build_dir: Path, abi3: bool + ) -> list[Path]: + from Cython.Build import cythonize + from setuptools import Distribution, Extension + + rtree_src = SRC / "spatial_graph" / "_rtree" + win = sys.platform == "win32" + extensions = [ + Extension( + f"{PKG.replace('/', '.')}.{name}", + sources=[str(pyx_dir / f"{name}.pyx")], + include_dirs=[str(rtree_src)], + extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], + define_macros=[ + *([("Py_LIMITED_API", ABI3_HEX)] if abi3 else []), + *([("RTREE_NOATOMICS", "1")] if win else []), + ], + py_limited_api=abi3, + ) + for name in names + ] + + out = build_dir / "lib" + dist = Distribution( + { + "name": "spatial_graph_prebuilt", + "ext_modules": cythonize( + extensions, language_level=3, quiet=True, nthreads=os.cpu_count() + ), + } + ) + cmd = dist.get_command_obj("build_ext") + cmd.build_lib = str(out) + cmd.build_temp = str(build_dir / "temp") + cmd.parallel = os.cpu_count() + cmd.ensure_finalized() + cmd.run() + + built_pkg = out.joinpath(*PKG.split("/")) + artifacts = sorted( + p for p in built_pkg.iterdir() if p.suffix in (".so", ".pyd") + ) + if len(artifacts) != len(names): + raise RuntimeError(f"expected {len(names)} modules, built {len(artifacts)}") + return artifacts diff --git a/pyproject.toml b/pyproject.toml index de66160..67d7c24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,20 @@ [build-system] -requires = ["hatchling", "hatch-vcs"] +requires = [ + "hatchling", + "hatch-vcs", + "Cython>=3.1", + "CT3>=3.3.3", + "numpy", # imported (not linked) while rendering wrappers + "setuptools>=75.8.0", +] build-backend = "hatchling.build" [tool.hatch.version] source = "vcs" +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + [project] name = "spatial-graph" dynamic = ["version"] diff --git a/src/spatial_graph/__init__.py b/src/spatial_graph/__init__.py index 4ec1dee..c1ab15a 100644 --- a/src/spatial_graph/__init__.py +++ b/src/spatial_graph/__init__.py @@ -1,4 +1,5 @@ from importlib.metadata import PackageNotFoundError, version +from typing import TYPE_CHECKING, Any try: __version__ = version("spatial_graph") @@ -6,10 +7,25 @@ __version__ = "unknown" -from ._graph import DiGraph, Graph, GraphBase from ._rtree import LineRTree, PointRTree -from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase -from ._util import create_graph + +if TYPE_CHECKING: + from ._graph import DiGraph, Graph, GraphBase + from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase + from ._util import create_graph + +# the graph half is always JIT-compiled, and importing it pulls in witty and +# Cheetah. Deferring it keeps `PointRTree`/`LineRTree` -- which ship prebuilt -- +# usable with numpy alone. +_LAZY = { + "DiGraph": "._graph", + "Graph": "._graph", + "GraphBase": "._graph", + "SpatialDiGraph": "._spatial_graph", + "SpatialGraph": "._spatial_graph", + "SpatialGraphBase": "._spatial_graph", + "create_graph": "._util", +} __all__ = [ "DiGraph", @@ -22,3 +38,15 @@ "SpatialGraphBase", "create_graph", ] + + +def __getattr__(name: str) -> Any: + if module := _LAZY.get(name): + import importlib + + return getattr(importlib.import_module(module, __name__), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return __all__ diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py new file mode 100644 index 0000000..17c349e --- /dev/null +++ b/src/spatial_graph/_rtree/_codegen.py @@ -0,0 +1,41 @@ +"""Rendering of the RTree pyx wrapper. + +Used on the JIT path and by the build hook, so prebuilt and JIT-compiled modules +are always generated from the same source. Requires Cheetah, and is therefore +imported lazily by `rtree.py`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from Cheetah.Template import Template + +from spatial_graph._dtypes import DType + +from ._naming import SRC_DIR + +if TYPE_CHECKING: + from .rtree import RTree + +TEMPLATE = SRC_DIR / "wrapper_template.pyx" + + +def build_wrapper( + cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int +) -> str: + """Render the pyx wrapper for the given tree parameters.""" + wrapper_template = Template( + file=str(TEMPLATE), + compilerSettings={"directiveStartToken": "%"}, + ) + wrapper_template.item_dtype = DType(item_dtype) + wrapper_template.coord_dtype = DType(coord_dtype) + wrapper_template.dims = dims + wrapper_template.c_distance_function = cls.c_distance_function + wrapper_template.pyx_item_t_declaration = cls.pyx_item_t_declaration + wrapper_template.c_item_t_declaration = cls.c_item_t_declaration + wrapper_template.c_converter_functions = cls.c_converter_functions + wrapper_template.c_equal_function = cls.c_equal_function + + return str(wrapper_template) diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py new file mode 100644 index 0000000..c048f65 --- /dev/null +++ b/src/spatial_graph/_rtree/_naming.py @@ -0,0 +1,49 @@ +"""Deterministic naming for prebuilt RTree extension modules. + +Shared by the runtime lookup and the build hook, so the two can never disagree. +Deliberately depends only on `_dtypes` -- it sits on the import path of every +`PointRTree`, including installs with neither Cheetah nor witty available. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import TYPE_CHECKING + +from spatial_graph._dtypes import DType + +if TYPE_CHECKING: + from .rtree import RTree + +SRC_DIR = Path(__file__).parent + +# subpackage holding ahead-of-time compiled modules; absent from pure-Python installs +PREBUILT_PACKAGE = f"{__package__}._prebuilt" + + +def _c_name(dtype: DType) -> str: + """Canonical, identifier-safe name for a dtype ("int64", "float", "int64x2").""" + base = dtype.base_c_type.removesuffix("_t") + return f"{base}x{dtype.size}" if dtype.is_array else base + + +def module_name(cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int) -> str: + """Deterministic module name for the given tree parameters. + + Dtypes are canonicalized (so `int` and `int64` agree) and spelled out for + readability. The trailing digest covers the C/pyx code `cls` injects into the + template, so a subclass with custom code can never be served a prebuilt + module compiled from different code. + """ + parts = ( + cls.pyx_item_t_declaration, + cls.c_item_t_declaration, + cls.c_converter_functions, + cls.c_equal_function, + cls.c_distance_function, + ) + digest = hashlib.sha256("\0".join(parts).encode()).hexdigest()[:8] + item = _c_name(DType(item_dtype)) + coord = _c_name(DType(coord_dtype)) + return f"rtree_{item}_{coord}_d{dims}_{digest}" diff --git a/src/spatial_graph/_rtree/_specs.py b/src/spatial_graph/_rtree/_specs.py new file mode 100644 index 0000000..cbfd352 --- /dev/null +++ b/src/spatial_graph/_rtree/_specs.py @@ -0,0 +1,40 @@ +"""The set of RTree variants compiled ahead of time into binary wheels. + +Only `PointRTree` is prebuilt by default: `LineRTree` is only ever used by +`SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it +would double the wheel size without removing anyone's compiler requirement. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, NamedTuple + +from .line_rtree import LineRTree +from .point_rtree import PointRTree + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .rtree import RTree + +ITEM_BASES = ("int64", "uint64") +COORD_DTYPES = ("float32", "float64") +DIMS = (2, 3, 4, 5) +PREBUILT_LINE_TREES = False + + +class Spec(NamedTuple): + cls: type[RTree] + item_dtype: str + coord_dtype: str + dims: int + + +def iter_specs() -> Iterator[Spec]: + """Yield every RTree variant that should be compiled into a wheel.""" + for base in ITEM_BASES: + for coord in COORD_DTYPES: + for dims in DIMS: + yield Spec(PointRTree, base, coord, dims) + if PREBUILT_LINE_TREES: + yield Spec(LineRTree, f"{base}[2]", coord, dims) diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 7bf280b..1d2aaba 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -1,51 +1,51 @@ from __future__ import annotations +import importlib +import os import sys -from pathlib import Path from typing import ClassVar import numpy as np -import witty -from Cheetah.Template import Template from spatial_graph._dtypes import DType +from ._naming import PREBUILT_PACKAGE, SRC_DIR, module_name + DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover EXTRA_COMPILE_ARGS = ["/O2"] else: EXTRA_COMPILE_ARGS = ["-O3", "-Wno-unreachable-code"] -SRC_DIR = Path(__file__).parent - -def _build_wrapper( +def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int -) -> str: - ############################################ - # create wrapper from template and compile # - ############################################ - - wrapper_template = Template( - file=str(SRC_DIR / "wrapper_template.pyx"), - compilerSettings={"directiveStartToken": "%"}, - ) - wrapper_template.item_dtype = DType(item_dtype) - wrapper_template.coord_dtype = DType(coord_dtype) - wrapper_template.dims = dims - wrapper_template.c_distance_function = cls.c_distance_function - wrapper_template.pyx_item_t_declaration = cls.pyx_item_t_declaration - wrapper_template.c_item_t_declaration = cls.c_item_t_declaration - wrapper_template.c_converter_functions = cls.c_converter_functions - wrapper_template.c_equal_function = cls.c_equal_function - - return str(wrapper_template) +) -> type | None: + """Return the ahead-of-time compiled tree class, or None if not shipped.""" + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return None + name = module_name(cls, item_dtype, coord_dtype, dims) + try: + module = importlib.import_module(f"{PREBUILT_PACKAGE}.{name}") + except ImportError: + return None + return module.RTree -def _compile_tree( +def _jit_compile_tree( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int ) -> type: - wrapper = _build_wrapper(cls, item_dtype, coord_dtype, dims) + """Compile a tree with the system C compiler. + + Only reached for dtype combinations not shipped prebuilt; Cheetah and witty + are imported here so neither is needed by installs that stay on the + prebuilt path. + """ + import witty + + from ._codegen import build_wrapper + + wrapper = build_wrapper(cls, item_dtype, coord_dtype, dims) module = witty.compile_cython( wrapper, depends_on=[ @@ -62,6 +62,15 @@ def _compile_tree( return module.RTree +def _compile_tree( + cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int +) -> type: + tree_cls = _load_prebuilt(cls, item_dtype, coord_dtype, dims) + if tree_cls is None: + tree_cls = _jit_compile_tree(cls, item_dtype, coord_dtype, dims) + return tree_cls + + class RTree: """A generic RTree implementation, compiled on-the-fly during instantiation. diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py new file mode 100644 index 0000000..f3d2013 --- /dev/null +++ b/tests/test_prebuilt.py @@ -0,0 +1,75 @@ +"""Tests for ahead-of-time compiled rtree modules. + +The `prebuilt` marked tests only mean something against an installed wheel; in a +source checkout there is no `_prebuilt` subpackage and they are skipped. +""" + +from __future__ import annotations + +import importlib.util + +import numpy as np +import pytest + +from spatial_graph import PointRTree +from spatial_graph._rtree._naming import PREBUILT_PACKAGE, module_name +from spatial_graph._rtree._specs import iter_specs +from spatial_graph._rtree.rtree import _load_prebuilt + +has_prebuilt = importlib.util.find_spec(PREBUILT_PACKAGE) is not None +requires_prebuilt = pytest.mark.skipif( + not has_prebuilt, reason="no prebuilt modules in this install" +) + + +@requires_prebuilt +@pytest.mark.parametrize("spec", list(iter_specs()), ids=str) +def test_every_declared_spec_is_shipped(spec): + """Every variant in `_specs` must actually resolve to a prebuilt module.""" + assert _load_prebuilt(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + + +@requires_prebuilt +def test_prebuilt_is_used_and_correct(): + tree = PointRTree("int64", "float32", 3) + assert "_prebuilt" in type(tree._ctree).__module__ + + items = np.array([10, 20, 30], dtype="int64") + points = np.ascontiguousarray([[0, 0, 0], [1, 1, 1], [9, 9, 9]], dtype="float32") + tree.insert_point_items(items, points) + + lo, hi = np.array([0, 0, 0], "float32"), np.array([2, 2, 2], "float32") + assert sorted(tree.search(lo, hi).ravel().tolist()) == [10, 20] + assert tree.nearest(np.array([8.9, 8.9, 8.9], "float32"), 1).ravel()[0] == 30 + + +def test_dtype_aliases_share_a_module(): + """`int`/`int64` and `float32`/`float` must not compile separate modules.""" + assert module_name(PointRTree, "int", "float32", 3) == module_name( + PointRTree, "int64", "float", 3 + ) + + +@pytest.mark.parametrize( + ("item_dtype", "coord_dtype", "dims"), + [("int32", "float32", 3), ("int64", "float32", 99)], +) +def test_unlisted_combination_falls_back_to_jit(item_dtype, coord_dtype, dims): + assert _load_prebuilt(PointRTree, item_dtype, coord_dtype, dims) is None + + +def test_subclass_with_custom_code_is_not_served_a_prebuilt_module(): + class CustomEquality(PointRTree): + c_equal_function = """ +inline bool equal(const item_t a, const item_t b) { return a == b; } +""" + + assert module_name(CustomEquality, "int64", "float32", 3) != module_name( + PointRTree, "int64", "float32", 3 + ) + assert _load_prebuilt(CustomEquality, "int64", "float32", 3) is None + + +def test_no_prebuilt_env_var_forces_jit(monkeypatch): + monkeypatch.setenv("SPATIAL_GRAPH_NO_PREBUILT", "1") + assert _load_prebuilt(PointRTree, "int64", "float32", 3) is None From 60ddc4085a5305b3624b5758a5fa1238d67e8596 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:03:01 +0200 Subject: [PATCH 2/6] drop Python 3.10; abi3 build is now unconditional --- hatch_build.py | 33 +++++++++++++++++---------------- pyproject.toml | 8 +++----- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index af48e78..45e7d9a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -19,7 +19,10 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface -# oldest CPython with the buffer protocol (memoryviews) in the limited API +# The wrappers pass numpy arrays as typed memoryviews, which compile to +# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 +# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), +# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. ABI3_MIN = (3, 11) ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" @@ -55,16 +58,19 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): return - # 3.10 lacks the buffer protocol in the limited API, so it gets a plain - # version-specific wheel; 3.11+ all share one abi3 wheel per platform. - abi3 = sys.version_info >= ABI3_MIN + if sys.version_info < ABI3_MIN: + raise RuntimeError( + f"building spatial-graph wheels requires Python >= " + f"{'.'.join(map(str, ABI3_MIN))}; the resulting abi3 wheel then " + f"covers every supported CPython." + ) _stub_spatial_graph_package() from spatial_graph._rtree._codegen import build_wrapper from spatial_graph._rtree._naming import module_name from spatial_graph._rtree._specs import iter_specs - build_dir = ROOT / "build" / "prebuilt" / f"{_platform_tag()}-{abi3}" + build_dir = ROOT / "build" / "prebuilt" / _platform_tag() pyx_dir = build_dir / "pyx" pyx_dir.mkdir(parents=True, exist_ok=True) @@ -81,7 +87,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: names.append(name) try: - built = self._compile(pyx_dir, names, build_dir, abi3) + built = self._compile(pyx_dir, names, build_dir) except Exception as e: # Installing from an sdist on a machine with no usable compiler must # keep working: fall back to a pure-Python wheel that JIT-compiles on @@ -103,14 +109,9 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: force_include[str(artifact)] = f"{PKG}/{artifact.name}" build_data["pure_python"] = False - if abi3: - build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" - else: - build_data["infer_tag"] = True - - def _compile( - self, pyx_dir: Path, names: list[str], build_dir: Path, abi3: bool - ) -> list[Path]: + build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" + + def _compile(self, pyx_dir: Path, names: list[str], build_dir: Path) -> list[Path]: from Cython.Build import cythonize from setuptools import Distribution, Extension @@ -123,10 +124,10 @@ def _compile( include_dirs=[str(rtree_src)], extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], define_macros=[ - *([("Py_LIMITED_API", ABI3_HEX)] if abi3 else []), + ("Py_LIMITED_API", ABI3_HEX), *([("RTREE_NOATOMICS", "1")] if win else []), ], - py_limited_api=abi3, + py_limited_api=True, ) for name in names ] diff --git a/pyproject.toml b/pyproject.toml index 67d7c24..b4356bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "spatial-graph" dynamic = ["version"] description = "A spatial graph datastructure for python." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" license = { text = "MIT" } authors = [ { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, @@ -30,7 +30,6 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -43,8 +42,7 @@ dependencies = [ "numpy>=2.3.2; python_version >= '3.14'", "numpy>=2.1.0; python_version >= '3.13'", "numpy>=1.26.0; python_version >= '3.12'", - "numpy>=1.23.2; python_version >= '3.11'", - "numpy>=1.21.2", + "numpy>=1.23.2", "setuptools>=75.8.0", "typing_extensions>=4.5.0", # witty<=0.3.1 imports it without declaring it ] @@ -74,7 +72,7 @@ homepage = "https://github.com/funkelab/spatial_graph" repository = "https://github.com/funkelab/spatial_graph" [tool.ruff] -target-version = "py310" +target-version = "py311" line-length = 88 fix = true unsafe-fixes = true From 8a993b57d57982477ee275e0e5d8eda6e4082fc0 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:33:58 +0200 Subject: [PATCH 3/6] switch build backend to setuptools; merge _specs into _codegen --- hatch_build.py | 157 ------------------ pyproject.toml | 20 ++- setup.py | 125 ++++++++++++++ src/spatial_graph/_rtree/_codegen.py | 45 ++++- src/spatial_graph/_rtree/_naming.py | 7 +- .../_rtree/_prebuilt/__init__.py | 5 + src/spatial_graph/_rtree/_specs.py | 40 ----- src/spatial_graph/_rtree/rtree.py | 5 +- tests/test_prebuilt.py | 10 +- 9 files changed, 190 insertions(+), 224 deletions(-) delete mode 100644 hatch_build.py create mode 100644 setup.py create mode 100644 src/spatial_graph/_rtree/_prebuilt/__init__.py delete mode 100644 src/spatial_graph/_rtree/_specs.py diff --git a/hatch_build.py b/hatch_build.py deleted file mode 100644 index 45e7d9a..0000000 --- a/hatch_build.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Compile RTree variants ahead of time into stable-ABI (abi3) wheels. - -Renders the same pyx wrappers the runtime would JIT-compile (via -`_rtree._codegen`) for every variant in `_rtree._specs`, builds them against -`Py_LIMITED_API`, and force-includes the result as -`spatial_graph/_rtree/_prebuilt/`. One wheel per platform then covers every -supported CPython, and users never need a C compiler for those variants. - -Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead. -""" - -from __future__ import annotations - -import os -import sys -import sysconfig -from pathlib import Path -from typing import Any - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface - -# The wrappers pass numpy arrays as typed memoryviews, which compile to -# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 -# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), -# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. -ABI3_MIN = (3, 11) -ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" - -ROOT = Path(__file__).parent -SRC = ROOT / "src" -PKG = "spatial_graph/_rtree/_prebuilt" - - -def _platform_tag() -> str: - return sysconfig.get_platform().replace("-", "_").replace(".", "_") - - -def _stub_spatial_graph_package() -> None: - """Make `spatial_graph.*` submodules importable without running its `__init__`. - - `spatial_graph/__init__.py` pulls in the graph half, and with it witty; the - build only needs the rtree codegen, so we register a bare namespace package - pointing at the source tree instead. - """ - import types - - pkg = types.ModuleType("spatial_graph") - pkg.__path__ = [str(SRC / "spatial_graph")] # type: ignore[attr-defined] - sys.modules.setdefault("spatial_graph", pkg) - - -class PrebuiltRTreeHook(BuildHookInterface): - PLUGIN_NAME = "prebuilt-rtree" - - def initialize(self, version: str, build_data: dict[str, Any]) -> None: - if self.target_name != "wheel": - return - if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): - return - - if sys.version_info < ABI3_MIN: - raise RuntimeError( - f"building spatial-graph wheels requires Python >= " - f"{'.'.join(map(str, ABI3_MIN))}; the resulting abi3 wheel then " - f"covers every supported CPython." - ) - - _stub_spatial_graph_package() - from spatial_graph._rtree._codegen import build_wrapper - from spatial_graph._rtree._naming import module_name - from spatial_graph._rtree._specs import iter_specs - - build_dir = ROOT / "build" / "prebuilt" / _platform_tag() - pyx_dir = build_dir / "pyx" - pyx_dir.mkdir(parents=True, exist_ok=True) - - names = [] - for spec in iter_specs(): - name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) - source = build_wrapper( - spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims - ) - path = pyx_dir / f"{name}.pyx" - # only rewrite when changed, so cythonize can skip unchanged variants - if not path.is_file() or path.read_text() != source: - path.write_text(source) - names.append(name) - - try: - built = self._compile(pyx_dir, names, build_dir) - except Exception as e: - # Installing from an sdist on a machine with no usable compiler must - # keep working: fall back to a pure-Python wheel that JIT-compiles on - # first use, exactly as before prebuilding existed. CI sets - # SPATIAL_GRAPH_REQUIRE_PREBUILT so this can never pass silently there. - if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): - raise - self.app.display_warning( - f"Could not prebuild rtree modules ({e}); building a pure-Python " - "wheel. A C compiler will be needed the first time an RTree is used." - ) - return - - force_include = build_data.setdefault("force_include", {}) - init = build_dir / "__init__.py" - init.write_text("") - force_include[str(init)] = f"{PKG}/__init__.py" - for artifact in built: - force_include[str(artifact)] = f"{PKG}/{artifact.name}" - - build_data["pure_python"] = False - build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" - - def _compile(self, pyx_dir: Path, names: list[str], build_dir: Path) -> list[Path]: - from Cython.Build import cythonize - from setuptools import Distribution, Extension - - rtree_src = SRC / "spatial_graph" / "_rtree" - win = sys.platform == "win32" - extensions = [ - Extension( - f"{PKG.replace('/', '.')}.{name}", - sources=[str(pyx_dir / f"{name}.pyx")], - include_dirs=[str(rtree_src)], - extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], - define_macros=[ - ("Py_LIMITED_API", ABI3_HEX), - *([("RTREE_NOATOMICS", "1")] if win else []), - ], - py_limited_api=True, - ) - for name in names - ] - - out = build_dir / "lib" - dist = Distribution( - { - "name": "spatial_graph_prebuilt", - "ext_modules": cythonize( - extensions, language_level=3, quiet=True, nthreads=os.cpu_count() - ), - } - ) - cmd = dist.get_command_obj("build_ext") - cmd.build_lib = str(out) - cmd.build_temp = str(build_dir / "temp") - cmd.parallel = os.cpu_count() - cmd.ensure_finalized() - cmd.run() - - built_pkg = out.joinpath(*PKG.split("/")) - artifacts = sorted( - p for p in built_pkg.iterdir() if p.suffix in (".so", ".pyd") - ) - if len(artifacts) != len(names): - raise RuntimeError(f"expected {len(names)} modules, built {len(artifacts)}") - return artifacts diff --git a/pyproject.toml b/pyproject.toml index b4356bc..e223f92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,21 @@ [build-system] requires = [ - "hatchling", - "hatch-vcs", + "setuptools>=77", + "setuptools-scm>=8", "Cython>=3.1", "CT3>=3.3.3", - "numpy", # imported (not linked) while rendering wrappers - "setuptools>=75.8.0", + "numpy", # imported (not linked) while rendering the wrappers ] -build-backend = "hatchling.build" +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] -[tool.hatch.version] -source = "vcs" +[tool.setuptools.packages.find] +where = ["src"] -[tool.hatch.build.targets.wheel.hooks.custom] -path = "hatch_build.py" +[tool.setuptools.package-data] +# the JIT fallback compiles from these at runtime, so they must ship in the wheel +"*" = ["py.typed", "*.pyx", "*.c", "*.h", "LICENSE*", "*.md"] [project] name = "spatial-graph" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..67c98d7 --- /dev/null +++ b/setup.py @@ -0,0 +1,125 @@ +"""Compile RTree variants ahead of time into a stable-ABI (abi3) wheel. + +Renders the same pyx wrappers the runtime would JIT-compile (via +`_rtree._codegen`) for every variant in `iter_specs()`, so prebuilt and +JIT-compiled modules can never disagree. One wheel per platform then covers +every supported CPython, and users never need a C compiler for those variants. + +Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead, or +`SPATIAL_GRAPH_REQUIRE_PREBUILT=1` (as CI does) to turn a failure to compile +into a hard error rather than a silent fall back to JIT. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import warnings +from pathlib import Path + +from setuptools import Extension, setup + +ROOT = Path(__file__).parent +SRC = ROOT / "src" +PREBUILT_PKG = "spatial_graph._rtree._prebuilt" + +# The wrappers pass numpy arrays as typed memoryviews, which compile to +# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 +# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), +# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. +ABI3_MIN = (3, 11) +ABI3_TAG = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}" +ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" + +WIN = sys.platform == "win32" + + +def prebuilt_extensions() -> list[Extension]: + """Render every prebuilt RTree variant and declare it as an extension.""" + from Cython.Build import cythonize + + sys.path.insert(0, str(SRC)) + from spatial_graph._rtree._codegen import build_wrapper, iter_specs + from spatial_graph._rtree._naming import module_name + + pyx_dir = ROOT / "build" / "prebuilt-pyx" + pyx_dir.mkdir(parents=True, exist_ok=True) + + extensions = [] + for spec in iter_specs(): + name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + source = build_wrapper(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + path = pyx_dir / f"{name}.pyx" + # only rewrite when changed, so cythonize can skip unchanged variants + if not path.is_file() or path.read_text() != source: + path.write_text(source) + extensions.append( + Extension( + f"{PREBUILT_PKG}.{name}", + sources=[str(path)], + include_dirs=[str(SRC / "spatial_graph" / "_rtree")], + extra_compile_args=["/O2"] if WIN else ["-O3", "-Wno-unreachable-code"], + define_macros=[ + ("Py_LIMITED_API", ABI3_HEX), + *([("RTREE_NOATOMICS", "1")] if WIN else []), + ], + py_limited_api=True, + ) + ) + + return cythonize( + extensions, + language_level=3, + quiet=True, + nthreads=0 if WIN else os.cpu_count(), + ) + + +def can_compile() -> bool: + """Whether this machine can build a C extension at all.""" + from distutils.ccompiler import new_compiler + from distutils.sysconfig import customize_compiler + + compiler = new_compiler() + customize_compiler(compiler) # picks up CC/CFLAGS, as build_ext does + with tempfile.TemporaryDirectory() as tmp: + probe = Path(tmp, "probe.c") + probe.write_text("int main(void) { return 0; }\n") + try: + compiler.compile([str(probe)], output_dir=tmp) + except Exception: + return False + return True + + +def should_prebuild() -> bool: + """Whether to compile prebuilt variants into this wheel.""" + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return False + if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + return True # CI: never let a build silently degrade + if can_compile(): + return True + # Installing from an sdist without a compiler must keep working: fall back to + # a pure-Python wheel that JIT-compiles on first use, as it did before + # prebuilding existed. + warnings.warn( + "No usable C compiler found; building spatial-graph without prebuilt " + "rtree modules. A C compiler will be needed the first time an RTree is " + "used.", + stacklevel=1, + ) + return False + + +if should_prebuild(): + setup( + ext_modules=prebuilt_extensions(), + options={ + "bdist_wheel": {"py_limited_api": ABI3_TAG}, + "build_ext": {"parallel": os.cpu_count()}, + }, + ) +else: + setup(ext_modules=[]) diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index 17c349e..c1d401e 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -1,24 +1,55 @@ -"""Rendering of the RTree pyx wrapper. +"""What RTree variants get prebuilt, and how their pyx wrappers are rendered. -Used on the JIT path and by the build hook, so prebuilt and JIT-compiled modules -are always generated from the same source. Requires Cheetah, and is therefore -imported lazily by `rtree.py`. +Used on the JIT path and by `setup.py`, so prebuilt and JIT-compiled modules are +always generated from the same source. Requires Cheetah, and is therefore +imported lazily by `rtree.py` -- installs that stay on the prebuilt path need +neither Cheetah nor witty. """ from __future__ import annotations -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple from Cheetah.Template import Template from spatial_graph._dtypes import DType -from ._naming import SRC_DIR +from .line_rtree import LineRTree +from .point_rtree import PointRTree if TYPE_CHECKING: + from collections.abc import Iterator + from .rtree import RTree -TEMPLATE = SRC_DIR / "wrapper_template.pyx" +TEMPLATE = Path(__file__).parent / "wrapper_template.pyx" + +# Variants compiled ahead of time into binary wheels. Only `PointRTree` by +# default: `LineRTree` is only ever used by `SpatialGraph`, whose graph half is +# JIT-compiled regardless, so prebuilding it would double the wheel size without +# removing anyone's compiler requirement. +ITEM_BASES = ("int64", "uint64") +COORD_DTYPES = ("float32", "float64") +DIMS = (2, 3, 4, 5) +PREBUILT_LINE_TREES = False + + +class Spec(NamedTuple): + cls: type[RTree] + item_dtype: str + coord_dtype: str + dims: int + + +def iter_specs() -> Iterator[Spec]: + """Yield every RTree variant that should be compiled into a wheel.""" + for base in ITEM_BASES: + for coord in COORD_DTYPES: + for dims in DIMS: + yield Spec(PointRTree, base, coord, dims) + if PREBUILT_LINE_TREES: + yield Spec(LineRTree, f"{base}[2]", coord, dims) def build_wrapper( diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index c048f65..dcf34eb 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -1,6 +1,6 @@ """Deterministic naming for prebuilt RTree extension modules. -Shared by the runtime lookup and the build hook, so the two can never disagree. +Shared by the runtime lookup and `setup.py`, so the two can never disagree. Deliberately depends only on `_dtypes` -- it sits on the import path of every `PointRTree`, including installs with neither Cheetah nor witty available. """ @@ -8,7 +8,6 @@ from __future__ import annotations import hashlib -from pathlib import Path from typing import TYPE_CHECKING from spatial_graph._dtypes import DType @@ -16,9 +15,7 @@ if TYPE_CHECKING: from .rtree import RTree -SRC_DIR = Path(__file__).parent - -# subpackage holding ahead-of-time compiled modules; absent from pure-Python installs +# subpackage holding ahead-of-time compiled modules; empty in a source checkout PREBUILT_PACKAGE = f"{__package__}._prebuilt" diff --git a/src/spatial_graph/_rtree/_prebuilt/__init__.py b/src/spatial_graph/_rtree/_prebuilt/__init__.py new file mode 100644 index 0000000..28a9aa6 --- /dev/null +++ b/src/spatial_graph/_rtree/_prebuilt/__init__.py @@ -0,0 +1,5 @@ +"""Ahead-of-time compiled RTree modules, populated at build time by `setup.py`. + +Empty in a plain source checkout: `_load_prebuilt` then finds nothing and every +tree is JIT-compiled, exactly as before prebuilding existed. +""" diff --git a/src/spatial_graph/_rtree/_specs.py b/src/spatial_graph/_rtree/_specs.py deleted file mode 100644 index cbfd352..0000000 --- a/src/spatial_graph/_rtree/_specs.py +++ /dev/null @@ -1,40 +0,0 @@ -"""The set of RTree variants compiled ahead of time into binary wheels. - -Only `PointRTree` is prebuilt by default: `LineRTree` is only ever used by -`SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it -would double the wheel size without removing anyone's compiler requirement. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, NamedTuple - -from .line_rtree import LineRTree -from .point_rtree import PointRTree - -if TYPE_CHECKING: - from collections.abc import Iterator - - from .rtree import RTree - -ITEM_BASES = ("int64", "uint64") -COORD_DTYPES = ("float32", "float64") -DIMS = (2, 3, 4, 5) -PREBUILT_LINE_TREES = False - - -class Spec(NamedTuple): - cls: type[RTree] - item_dtype: str - coord_dtype: str - dims: int - - -def iter_specs() -> Iterator[Spec]: - """Yield every RTree variant that should be compiled into a wheel.""" - for base in ITEM_BASES: - for coord in COORD_DTYPES: - for dims in DIMS: - yield Spec(PointRTree, base, coord, dims) - if PREBUILT_LINE_TREES: - yield Spec(LineRTree, f"{base}[2]", coord, dims) diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 1d2aaba..8afb50e 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -3,13 +3,14 @@ import importlib import os import sys +from pathlib import Path from typing import ClassVar import numpy as np from spatial_graph._dtypes import DType -from ._naming import PREBUILT_PACKAGE, SRC_DIR, module_name +from ._naming import PREBUILT_PACKAGE, module_name DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover @@ -17,6 +18,8 @@ else: EXTRA_COMPILE_ARGS = ["-O3", "-Wno-unreachable-code"] +SRC_DIR = Path(__file__).parent + def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index f3d2013..ec8c775 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -6,17 +6,17 @@ from __future__ import annotations -import importlib.util - import numpy as np import pytest from spatial_graph import PointRTree -from spatial_graph._rtree._naming import PREBUILT_PACKAGE, module_name -from spatial_graph._rtree._specs import iter_specs +from spatial_graph._rtree._codegen import iter_specs +from spatial_graph._rtree._naming import module_name from spatial_graph._rtree.rtree import _load_prebuilt -has_prebuilt = importlib.util.find_spec(PREBUILT_PACKAGE) is not None +# the `_prebuilt` package always exists but is empty in a source checkout, so +# probe for a real module rather than for the package +has_prebuilt = _load_prebuilt(PointRTree, "int64", "float32", 2) is not None requires_prebuilt = pytest.mark.skipif( not has_prebuilt, reason="no prebuilt modules in this install" ) From 4fac7b076ecb704d6c99a8673d573b69ae85095e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:42:17 +0200 Subject: [PATCH 4/6] ci: drop 3.10, test the built wheel instead of an editable install --- .github/workflows/ci.yml | 16 +++++++++++----- tests/test_prebuilt.py | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f73cf42..4f2da43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,17 +29,19 @@ jobs: matrix: # ubuntu: full python range x both resolutions os: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] resolution: [lowest-direct, highest] # windows/macos: only the endpoints, highest resolution include: - - { os: windows-latest, python-version: "3.10", resolution: highest } + - { os: windows-latest, python-version: "3.11", resolution: highest } - { os: windows-latest, python-version: "3.14", resolution: highest } - - { os: macos-latest, python-version: "3.10", resolution: highest } + - { os: macos-latest, python-version: "3.11", resolution: highest } - { os: macos-latest, python-version: "3.14", resolution: highest } env: UV_RESOLUTION: ${{ matrix.resolution }} + # a build that silently falls back to pure-Python must fail, not go green + SPATIAL_GRAPH_REQUIRE_PREBUILT: "1" steps: - uses: actions/checkout@v4 @@ -48,8 +50,12 @@ jobs: python-version: ${{ matrix.python-version }} enable-cache: true cache-dependency-glob: "**/pyproject.toml" + # --no-editable so we test the built wheel, prebuilt rtree modules and all, + # rather than an editable install of src/ + - name: Install as a built wheel + run: uv sync --no-dev --group test --no-editable - name: Test with coverage - run: uv run --no-dev --group test pytest -v --cov=spatial_graph --cov-report=xml + run: uv run --no-sync pytest -v --cov=spatial_graph --cov-report=xml - uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -66,7 +72,7 @@ jobs: enable-cache: true - name: install - run: uv sync --no-dev --group test-codspeed + run: uv sync --no-dev --group test-codspeed --no-editable - name: Run benchmarks uses: CodSpeedHQ/action@v3 diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index ec8c775..5ffa2b3 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -1,11 +1,15 @@ """Tests for ahead-of-time compiled rtree modules. -The `prebuilt` marked tests only mean something against an installed wheel; in a -source checkout there is no `_prebuilt` subpackage and they are skipped. +The `requires_prebuilt` tests only mean something against an install that +actually shipped them, and are skipped otherwise -- except when +`SPATIAL_GRAPH_REQUIRE_PREBUILT` is set (as CI does), where their absence is +the very regression we want to catch. """ from __future__ import annotations +import os + import numpy as np import pytest @@ -22,10 +26,17 @@ ) +def test_prebuilt_modules_were_shipped(): + """Guard against a wheel that silently degraded to pure Python.""" + if not os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + pytest.skip("SPATIAL_GRAPH_REQUIRE_PREBUILT not set") + assert has_prebuilt, "install shipped no prebuilt rtree modules" + + @requires_prebuilt @pytest.mark.parametrize("spec", list(iter_specs()), ids=str) def test_every_declared_spec_is_shipped(spec): - """Every variant in `_specs` must actually resolve to a prebuilt module.""" + """Every variant in `iter_specs` must actually resolve to a prebuilt module.""" assert _load_prebuilt(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) From 2049377a5e63a01a647b96158a1a129469f105b4 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 16:22:03 +0200 Subject: [PATCH 5/6] ci: build release wheels with cibuildwheel; prebuild LineRTree too --- .github/workflows/ci.yml | 106 ++++++++++++++++++++++++--- pyproject.toml | 10 +++ src/spatial_graph/_rtree/_codegen.py | 10 +-- 3 files changed, 111 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f2da43..8debc4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,9 +79,97 @@ jobs: with: run: uv run pytest -W ignore --codspeed -v --color=yes + # One abi3 wheel per platform, covering every supported CPython. Also the only + # thing that produces PyPI-acceptable manylinux tags -- `uv build` alone emits + # `linux_x86_64`, which PyPI rejects. + build-wheels: + name: Wheels ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest # manylinux x86_64 + - ubuntu-24.04-arm # manylinux aarch64 + - windows-latest # win_amd64 + - macos-13 # macOS x86_64 + - macos-latest # macOS arm64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # setuptools-scm needs the tags + - uses: pypa/cibuildwheel@v4.1.1 + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + + build-sdist: + name: Sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v6 + - run: uv build --sdist + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + # The claim this whole design rests on: one cp311-abi3 wheel runs on every + # supported CPython, with no compiler and no witty. + test-abi3-wheel: + name: abi3 wheel on py${{ matrix.python-version }} + needs: build-wheels + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + env: + SPATIAL_GRAPH_REQUIRE_PREBUILT: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-latest + path: wheelhouse + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install the wheel with numpy alone + run: | + uv venv + uv pip install numpy + uv pip install --no-deps wheelhouse/*.whl + - name: Prebuilt rtrees must work without witty, Cheetah or a compiler + run: | + uv run --no-sync python -c " + import sys, numpy as np + try: + import witty; sys.exit('witty present; test is not conclusive') + except ImportError: pass + from spatial_graph import PointRTree + t = PointRTree('int64', 'float32', 3) + t.insert_point_items(np.array([1, 2], dtype='int64'), + np.ascontiguousarray([[0,0,0],[9,9,9]], dtype='float32')) + mod = type(t._ctree).__module__ + assert '_prebuilt' in mod, mod + found = t.search(np.array([0,0,0],'float32'), np.array([1,1,1],'float32')) + assert found.ravel().tolist() == [1], found + print('ok:', mod)" + # the test module imports the codegen (and so Cheetah), so pull the real + # dependency set back in before running the suite + - name: Run the prebuilt test suite against the wheel + run: | + uv pip install wheelhouse/*.whl pytest + uv run --no-sync pytest tests/test_prebuilt.py -v + deploy: name: Deploy - needs: test + needs: [test, test-abi3-wheel, build-sdist] if: success() && startsWith(github.ref, 'refs/tags/') && github.event_name != 'schedule' runs-on: ubuntu-latest @@ -90,17 +178,15 @@ jobs: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: - fetch-depth: 0 - - uses: astral-sh/setup-uv@v6 + pattern: wheels-* + path: dist + merge-multiple: true + - uses: actions/download-artifact@v4 with: - python-version: ${{ matrix.python-version }} - enable-cache: true - cache-dependency-glob: "**/pyproject.toml" - - - name: 👷 Build - run: uv build + name: sdist + path: dist - name: 🚢 Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml index e223f92..0dda8d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,16 @@ docs = [ homepage = "https://github.com/funkelab/spatial_graph" repository = "https://github.com/funkelab/spatial_graph" +[tool.cibuildwheel] +# a single abi3 build per platform covers every supported CPython +build = "cp311-*" +# never let a wheel silently degrade to pure Python +environment = { SPATIAL_GRAPH_REQUIRE_PREBUILT = "1" } +test-groups = ["test"] +# these exercise the prebuilt modules in the repaired wheel without needing a +# compiler; cross-version and numpy-only checks live in the CI workflow +test-command = "pytest {project}/tests/test_prebuilt.py -q" + [tool.ruff] target-version = "py311" line-length = 88 diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index c1d401e..3b30f35 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -25,14 +25,14 @@ TEMPLATE = Path(__file__).parent / "wrapper_template.pyx" -# Variants compiled ahead of time into binary wheels. Only `PointRTree` by -# default: `LineRTree` is only ever used by `SpatialGraph`, whose graph half is -# JIT-compiled regardless, so prebuilding it would double the wheel size without -# removing anyone's compiler requirement. +# Variants compiled ahead of time into binary wheels. `PointRTree` is what makes +# a compiler unnecessary for rtree-only users; `LineRTree` is only reached via +# `SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it +# saves first-use compile time rather than removing a requirement. ITEM_BASES = ("int64", "uint64") COORD_DTYPES = ("float32", "float64") DIMS = (2, 3, 4, 5) -PREBUILT_LINE_TREES = False +PREBUILT_LINE_TREES = True class Spec(NamedTuple): From a74fd2a8b3edb3c9c4152df687fdce2b91c65812 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 17:44:16 +0200 Subject: [PATCH 6/6] ci: update macOS runner version to 15 for compatibility --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8debc4b..40e40e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: - ubuntu-latest # manylinux x86_64 - ubuntu-24.04-arm # manylinux aarch64 - windows-latest # win_amd64 - - macos-13 # macOS x86_64 + - macos-15-intel # macOS x86_64 - macos-latest # macOS arm64 steps: - uses: actions/checkout@v4