negpy_core: CPU kernels in Rust, no Numba, no cv2 in the app

.github/workflows/ci.yml

@@ -5,24 +5,40 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+  # mz/core runs on demand: gh workflow run ci.yml --ref mz/core
+  workflow_dispatch:
 
 jobs:
   test:
     name: Test and Lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
 
       - name: Set up Python
         run: uv python install 3.13
 
+      - name: Set up Rust
+        # Installs the toolchain pinned in negpy_core/rust-toolchain.toml.
+        working-directory: negpy_core
+        run: rustup toolchain install
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: negpy_core
+
+      - name: Rust licenses and bans (cargo-deny)
+        uses: EmbarkStudios/cargo-deny-action@v2
+        with:
+          manifest-path: negpy_core/Cargo.toml
+
       - name: Install System Dependencies (Linux)
         if: runner.os == 'Linux'
         run: |
@@ -34,6 +50,10 @@ jobs:
         # them those backend tests importorskip away.
         run: uv sync --group dev --group plustek --group pieusb
 
+      - name: Rust checks (fmt, clippy, test)
+        # Under uv run, PyO3's build script finds the project's Python 3.13, not the runner's.
+        run: uv run --no-sync make rust
+
       - name: Run Linting (Ruff)
         run: uv run ruff check .
 
@@ -58,8 +78,55 @@ jobs:
         run: uv run pytest tests/ --cov=negpy --cov-report=term-missing --cov-report=xml
 
       - name: Upload Coverage Report
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: coverage
           path: coverage.xml
           retention-days: 7
+
+  native:
+    # negpy_core must give the same bits on every release platform: run the Rust tests and
+    # the pytest oracles of the ported functions on each one.
+    name: Native (${{ matrix.os }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-14, macos-15-intel]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/[email protected]
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv python install 3.13
+
+      - name: Set up Rust
+        working-directory: negpy_core
+        run: rustup toolchain install
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: negpy_core
+
+      - name: Rust tests
+        working-directory: negpy_core
+        run: cargo test
+
+      - name: Install System Dependencies (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libegl1 libxkbcommon-x11-0 libdbus-1-3 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 x11-utils libgl1
+
+      - name: Install Dependencies
+        # The plustek group brings opencv, the oracle the ported-function tests compare against.
+        run: uv sync --group dev --group plustek
+
+      - name: Ported-function oracles
+        run: uv run pytest tests/test_percentiles.py tests/test_core_same_bits.py tests/test_core_resize.py tests/test_core_filter.py tests/test_core_morph.py tests/test_core_gray.py tests/test_core_contours.py tests/test_core_shapes.py tests/test_core_components.py tests/test_core_color.py tests/test_core_warp.py tests/test_core_lab.py tests/test_core_altprocess.py tests/test_core_finish.py

.github/workflows/metrics.yml

@@ -48,10 +48,10 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
 
@@ -65,7 +65,7 @@ jobs:
 
       - name: Restore perf fixture cache
         id: cache-fixture
-        uses: actions/cache@v4
+        uses: actions/cache@v6
         with:
           path: ~/.cache/negpy-metrics
           key: negpy-perf-raw-rawsamples-v1
@@ -105,7 +105,7 @@ jobs:
           test -f "$NEGPY_METRICS_OUT" && cat "$NEGPY_METRICS_OUT" || true
 
       - name: Upload metrics JSON
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: preview-metrics
           path: metrics-artifacts/preview_metrics.json
@@ -114,7 +114,7 @@ jobs:
       - name: Post PR comment
         if: github.event_name == 'pull_request'
         continue-on-error: true
-        uses: actions/github-script@v7
+        uses: actions/github-script@v9
         with:
           script: |
             const fs = require('fs');

.github/workflows/release.yml

@@ -17,21 +17,28 @@ on:
           - windows
           - linux
           - macos
+      publish:
+        description: 'Create the GitHub release (off for a test build)'
+        required: true
+        default: true
+        type: boolean
 
 jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
+      # Linux builds on a fixed Ubuntu: the AppImage needs the oldest glibc it can get, and
+      # ubuntu-latest moves to a newer release on its own.
       - id: set-matrix
         run: |
           if [ "${{ github.event.inputs.platform }}" == "all" ]; then
-            echo 'matrix=[{"os":"ubuntu-latest"},{"os":"windows-latest"},{"os":"macos-14","arch":"arm64"},{"os":"macos-15-intel","arch":"x86_64"}]' >> $GITHUB_OUTPUT
+            echo 'matrix=[{"os":"ubuntu-24.04"},{"os":"windows-latest"},{"os":"macos-14","arch":"arm64"},{"os":"macos-15-intel","arch":"x86_64"}]' >> $GITHUB_OUTPUT
           elif [ "${{ github.event.inputs.platform }}" == "windows" ]; then
             echo 'matrix=[{"os":"windows-latest"}]' >> $GITHUB_OUTPUT
           elif [ "${{ github.event.inputs.platform }}" == "linux" ]; then
-            echo 'matrix=[{"os":"ubuntu-latest"}]' >> $GITHUB_OUTPUT
+            echo 'matrix=[{"os":"ubuntu-24.04"}]' >> $GITHUB_OUTPUT
           elif [ "${{ github.event.inputs.platform }}" == "macos" ]; then
             echo 'matrix=[{"os":"macos-14","arch":"arm64"},{"os":"macos-15-intel","arch":"x86_64"}]' >> $GITHUB_OUTPUT
           fi
@@ -45,19 +52,28 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
 
       - name: Set up Python
         run: uv python install 3.13
 
+      - name: Set up Rust
+        # uv sync builds negpy_core with the toolchain pinned in negpy_core/rust-toolchain.toml.
+        working-directory: negpy_core
+        run: rustup toolchain install
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: negpy_core
+
       - name: Install System Dependencies (Linux)
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-24.04'
         run: |
           sudo apt-get update
           sudo apt-get install -y libfuse2 libxcb-cursor0 libxkbcommon-x11-0 libxcb-xinerama0 patchelf libsane-dev
@@ -66,7 +82,7 @@ jobs:
 
       - name: Install System Dependencies (macOS)
         if: startsWith(matrix.os, 'macos-')
-        run: brew install sane-backends llvm@20
+        run: brew install sane-backends
 
       - name: Install Packaging Tools (Windows)
         if: matrix.os == 'windows-latest'
@@ -80,18 +96,9 @@ jobs:
           }
 
       - name: Install Dependencies (Linux)
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-24.04'
         run: uv sync --all-groups
 
-      - name: Configure LLVM for llvmlite (macOS x86_64)
-        if: matrix.arch == 'x86_64'
-        shell: bash
-        run: |
-          LLVM20="$(brew --prefix llvm@20)"
-          echo "PATH=/usr/local/bin:${LLVM20}/bin:${PATH}" >> "$GITHUB_ENV"
-          echo "LLVM_CONFIG=${LLVM20}/bin/llvm-config" >> "$GITHUB_ENV"
-          echo "CMAKE_ARGS=-DLLVM_DIR=${LLVM20}/lib/cmake/llvm -DCMAKE_PREFIX_PATH=${LLVM20}" >> "$GITHUB_ENV"
-
       - name: Install Dependencies (macOS)
         if: startsWith(matrix.os, 'macos-')
         run: CFLAGS="-I$(brew --prefix)/include" LDFLAGS="-L$(brew --prefix)/lib" uv sync --all-groups
@@ -121,7 +128,7 @@ jobs:
         run: ls -R dist/
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: negpy-${{ matrix.os }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }}-${{ github.event.inputs.version }}
           if-no-files-found: error
@@ -133,20 +140,20 @@ jobs:
   publish:
     needs: build
     runs-on: ubuntu-latest
-    if: success()
+    if: success() && inputs.publish
     environment: production
     permissions:
       contents: write
     steps:
       - name: Download all artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           path: all-binaries
           pattern: negpy-*
           merge-multiple: true
 
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
         with:
           tag_name: ${{ github.event.inputs.version }}
           name: NegPy ${{ github.event.inputs.version }}

CLAUDE.md

@@ -15,22 +15,25 @@ Guidance for Claude Code in this repository. It holds rules and traps the code d
 ```bash
 make run          # Launch the desktop app
 make run-sim      # Launch with a simulated camera, Scanlight and film scanners
-make all          # lint + type check + tests (run before committing)
+make all          # format + Rust checks + lint + type check + tests (run before committing)
+make rust         # cargo fmt --check + clippy -D warnings + cargo test for negpy_core
+make replay       # replay recorded real calls through the current code, bit for bit (PHASE=N)
+make bench        # all-core speed of ported functions against the recorded Python (PHASE=N)
 make test         # pytest only
 make lint         # ruff check
 make type         # ty check (not mypy)
-make format       # ruff format + autofix
+make format       # cargo fmt + ruff format + autofix
 make translations # Extract translation sources and preserve existing translations
 
 # Single test
 uv run pytest tests/test_exposure_logic.py::test_name -v
 ```
 
-All commands run through `uv run`; never invoke pytest/ruff/ty directly. Run `make format` before committing; if it reformats unrelated files, commit those as lint fixes.
+All commands run through `uv run`; never invoke pytest/ruff/ty directly. `uv run` compiles `negpy_core` again after a Rust file changes (`tool.uv.cache-keys`). Run `make format` before committing; if it reformats unrelated files, commit those as lint fixes.
 
 ## Architecture
 
-NegPy is a film-negative processing desktop app (PyQt6 + WebGPU). Images flow through a multi-stage pipeline implemented twice, CPU (numpy/Numba) and GPU (WGSL via `wgpu`), which must stay in numerical parity.
+NegPy is a film-negative processing desktop app (PyQt6 + WebGPU). Images flow through a multi-stage pipeline implemented twice, CPU (numpy and `negpy_core`) and GPU (WGSL via `wgpu`), which must stay in numerical parity.
 
 ### Data model
 
@@ -50,6 +53,24 @@ Edits persist in SQLite (`edits.db`, keyed by content hash), optionally mirrored
 
 **Film edge print** (`services/film_edge/`): a stock's own marks, DX number and respool source (`prints_as`) are the `edge` object in `gear/film_stocks.json`, and each entry names its `source`. The `edge-print-from-photo` skill adds a stock from a photo of the film.
 
+### negpy_core (Rust)
+
+`negpy_core/` is a PyO3 extension (abi3-py313, maturin, a `uv` workspace member) that takes over CPU kernels one group at a time.
+
+- A ported function keeps its Python signature; the Python body becomes a call into `negpy_core`, and the Python implementation goes in the same change. No runtime switch, no fallback.
+- Core code in `src/<area>.rs` takes slices or `ndarray` views and plain numbers, never config. `src/py/<area>.rs` holds only the wrappers: dtype, shape and contiguity checks, `py::detach` around the work, errors as Python exceptions. No panic crosses into Python.
+- `cargo test` builds without Python: PyO3 is behind the `python` feature, which maturin turns on.
+- Pure Rust: no dependency compiles C or C++ (`deny.toml` bans `cc`, `cmake`, `pkg-config`, `bindgen`). Codecs that need C stay on their Python wheels. A new crate's license goes in `NOTICE.md`.
+- No `unsafe` except SIMD, each with a reason. Names mirror the Python names.
+- `mul_add` is one instruction only on a CPU with FMA; the release baseline has none, so a loop that uses it takes `#[multiversion(targets("x86_64+avx2+fma", "aarch64+neon"))]`. `mul_add` is exactly fused on every path, so the bits never depend on the CPU.
+- Hot loops: an element-wise pass goes through `par_zip` (rayon blocks; `par_iter_mut().zip` makes a task per element), and its closure is `move`. A capture read through a reference can alias the output, so LLVM reloads it per element and does not vectorize.
+- A float-to-int `as` cast saturates, and LLVM leaves such a loop scalar. Clamp first, then take the integer part as `convert.rs::trunc_bits` does (add 2^52, read the low bits).
+- **Bit-exact**: a port matches the Python result bit for bit (a zero's sign may differ where NumPy ranks -0.0 and 0.0 equal), checked by a pytest oracle against the numpy or `cv2` call it replaces. The `cv2` oracles run on Linux x86-64 only (`tests/core_oracle.py`: opencv-python's other builds differ in the last bits); `tests/test_core_same_bits.py` holds output hashes that every release OS must match (`ci.yml` job `native`). NumPy's rules to copy: the linear percentile (virtual index in f64, interpolation in f32, the `t >= 0.5` branch from the upper value); float32 sums in NumPy's order; NEP 50 promotion (a Python float joins float32 math as float32); Numba `fastmath` as f64 with `mul_add`; Python `round` ties to even. No fast-math. Threads only over independent outputs, or integer reductions, so the core count never changes a result. A reduction with a large accumulator takes one chunk per thread (rayon's `fold` makes one per split job) and enough work per chunk to repay clearing and merging it.
+- `negpy_core` holds every CPU kernel; NegPy has no Numba and imports no `cv2` (`tests/test_no_cv2.py`). opencv-python is only a dependency of the plustek extra and group, for pyopticfilm's pass alignment; the tests use it as their oracle through the plustek group. Every binding runs its work through `py::detach(py, n, f)`, never `py.detach`: it releases the GIL, keeps work under 64K elements on the calling thread and enters the all-core pool once per call (`tests/test_core_ir.py` checks). A thread marked by `negpy_core.set_background_thread(True)` (the thumbnail refresh) enters a pool of a quarter of the cores (at least two) instead, so the live preview keeps the rest. The app has no single-core mode.
+- `negpy_core/manifest.toml` lists every function to port (phase, status, `depends`). A port sets its entries to `ported`; `uv run python scripts/core_manifest.py` refreshes the rest after a change or a merge from `main`, and `tests/test_core_manifest.py` fails while it is stale.
+- **Replay**: before a group is ported, record its calls on main (`scripts/replay.py record FRAME... --phase N`, corpus in `$NEGPY_CORPUS`). The port passes `make replay` and `make bench` (never slower with every core in use, as users run the app). Both are local tools, not CI.
+- Transcendental functions (`log10`, `exp`, `pow`, ...) in array loops come from `src/mathf.rs` (branch-free, so they vectorize; `libm` is too slow for the speed gate), scalar ones from the `libm` crate. NumPy uses SVML on an AVX-512 CPU and libm elsewhere, so the Python bits already differ by CPU. A port that cannot be bit-exact writes an entry in `negpy_core/LEDGER.md` and a `compare` mode in the manifest (`rows`, `ulp=N`); the release bumps `BOUNDS_VERSION` and `THUMBNAIL_RENDER_VERSION` one time for the whole ledger.
+
 ### Pipeline
 
 - **CPU**: `DarkroomEngine.process()` (`negpy/services/rendering/engine.py`): base (geometry + normalization) → exposure (incl. dodge/burn) → clahe → lab → alt process → toning → crop → finish. The first four stages are cached per config-hash via `_run_stage()`. Base and exposure route on `process.path.render_path`: `PRINT` runs `features/exposure/`; `TRANSFER` and `POSITIVE` run `features/transparency/`. `features/exposure/` holds no slide branch and never imports `features/transparency/` (`tests/test_slide_separation.py`).

CONTRIBUTING.md

@@ -11,10 +11,12 @@ A 👀 reaction on your comment confirms that the command ran.
 
 ## Development Setup
 
-NegPy requires Python 3.13+ and `uv`.
+NegPy requires Python 3.13+, `uv` and a Rust toolchain.
 
 ### 1. Prerequisites
-Install [uv](https://docs.astral.sh/uv/getting-started/installation/).
+Install [uv](https://docs.astral.sh/uv/getting-started/installation/) and [rustup](https://rustup.rs/). `rustup` reads the toolchain version from `negpy_core/rust-toolchain.toml`.
+
+The `negpy_core/` folder is a Rust library with NegPy's CPU kernels. The first `uv sync` compiles it. After you edit a Rust file, the next `uv run` or `make test` compiles it again. `negpy_core` builds no C or C++ code: `cargo-deny` (`negpy_core/deny.toml`) rejects a dependency that does. Its tests compare against opencv, which only the `plustek` group installs; `make install` (`uv sync --all-groups`) brings it.
 
 Scanner support (optional):
 
@@ -108,6 +110,7 @@ backend are fake.
 - `negpy/kernel/`: core system services (Logging, Config, caching).
 - `negpy/services/`: high-level orchestration (Rendering engine, Export service).
 - `negpy/desktop/`: PyQt6 UI (View, Controller, Workers).
+- `negpy_core/`: Rust library for CPU kernels (`src/<area>.rs`), with its PyO3 wrappers in `src/py/`.
 - `tests/`: unit and integration tests.
 
 ## Coding Standards
@@ -149,8 +152,10 @@ Every developer command runs through `uv run`:
 - `make lint`: Ruff checks.
 - `make type`: `ty` type checks.
 - `make test`: unit tests.
-- `make format`: auto-format with Ruff.
-- `make all`: lint, type and test in sequence.
+- `make format`: auto-format with `cargo fmt` and Ruff.
+- `make rust`: `cargo fmt --check`, `cargo clippy` (warnings fail) and `cargo test` for `negpy_core`.
+- `make all`: format, Rust checks, lint, type and test in sequence.
+- `make replay` / `make bench`: check a `negpy_core` port against calls recorded on real frames, for identical results and speed. Record with `uv run python scripts/replay.py record FRAME...`; the corpus goes to `$NEGPY_CORPUS` (default `~/negpy-corpus`).
 - `make clean`: remove cache and build artifacts.
 - `make run-sim`: run with simulated scanning hardware (see [above](#running-without-scanning-hardware)).
 - `make clear-devhome`: delete the development user directory (see [above](#a-separate-user-directory-for-development)).