diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml new file mode 100644 index 00000000..8395c904 --- /dev/null +++ b/.github/workflows/node.yml @@ -0,0 +1,260 @@ +name: Node binding + +on: + push: + tags: ["node-v*"] + pull_request: + paths: + - "bindings/node/**" + - "crates/diffy-compare/**" + - "crates/diffy-diff/**" + - "crates/diffy-semantic/**" + - "crates/diffy-syntax/**" + - "tools/syntax-pack/**" + - "Cargo.lock" + - "Cargo.toml" + - "rust-toolchain.toml" + - ".github/workflows/node.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: node-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} + +env: + TREE_SITTER_CLI: 0.26.8 + NODE_VERSION: "22" + # Node 22 and Electron need macOS 11; C grammars and Rust agree on it. + MACOSX_DEPLOYMENT_TARGET: "11.0" + # The oldest glibc the Linux binaries may need (manylinux_2_28). + MAX_GLIBC: "2.28" + # .cargo/config.toml's macOS runner signs with a local identity CI lacks. + DIFFY_SKIP_CODESIGN: "1" + +jobs: + build: + name: build ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + target: aarch64-apple-darwin + os: macos-26 + # Cross-built on Apple silicon; tested with an x64 Node under Rosetta. + - platform: darwin-x64 + target: x86_64-apple-darwin + os: macos-26 + node-arch: x64 + # Built in manylinux_2_28 so the grammars' C and the Rust link see + # glibc 2.28 and nothing newer. + - platform: linux-x64-gnu + target: x86_64-unknown-linux-gnu + os: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 + - platform: linux-arm64-gnu + target: aarch64-unknown-linux-gnu + os: ubuntu-24.04-arm + container: quay.io/pypa/manylinux_2_28_aarch64 + - platform: win32-x64-msvc + target: x86_64-pc-windows-msvc + os: windows-2025 + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + timeout-minutes: 120 + defaults: + run: + shell: bash + env: + PLATFORM: ${{ matrix.platform }} + TARGET: ${{ matrix.target }} + steps: + # Before checkout, which needs git for the submodule. The image's GCC + # replaces .cargo/config.toml's clang (keeping -fuse-ld=lld) so the + # grammars and the Rust link use the image's glibc. libclang is for + # bindgen in tree-sitter-cli's QuickJS. + - name: Install build tools (manylinux) + if: matrix.container + run: | + dnf install -y git lld clang-devel + echo "CC=gcc" >> "$GITHUB_ENV" + echo "CARGO_TARGET_$(echo "$TARGET" | tr 'a-z-' 'A-Z_')_LINKER=gcc" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install rustup (manylinux) + if: matrix.container + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Install the pinned Rust toolchain + run: | + rustup toolchain install + rustup target add "$TARGET" + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: node-${{ matrix.target }}-v1 + save-if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }} + + - name: Install lld (macOS) + if: runner.os == 'macOS' + run: brew install lld + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + architecture: ${{ matrix.node-arch }} + + - name: Install tree-sitter CLI (npm) + if: ${{ !matrix.container }} + run: | + npm install --global "tree-sitter-cli@${TREE_SITTER_CLI}" + cli="$(npm root -g)/tree-sitter-cli/tree-sitter" + if [ "${{ runner.os }}" = Windows ]; then cli="$cli.exe"; fi + "$cli" --version + echo "TREE_SITTER=$cli" >> "$GITHUB_ENV" + + # npm's prebuilt CLI needs a newer glibc than the container has. + - uses: actions/cache@v4 + if: matrix.container + id: tree-sitter + with: + path: ~/tree-sitter-cli + key: tree-sitter-cli-${{ env.TREE_SITTER_CLI }}-${{ matrix.target }}-manylinux_2_28 + + - name: Install tree-sitter CLI (cargo) + if: matrix.container + run: | + if [ "${{ steps.tree-sitter.outputs.cache-hit }}" != true ]; then + cargo install --locked --root "$HOME/tree-sitter-cli" tree-sitter-cli --version "$TREE_SITTER_CLI" + fi + cli="$HOME/tree-sitter-cli/bin/tree-sitter" + "$cli" --version + echo "TREE_SITTER=$cli" >> "$GITHUB_ENV" + + - name: Bundle grammars + run: | + bundle="$GITHUB_WORKSPACE/target/syntax-bundle" + echo "DIFFY_SYNTAX_BUNDLE=$bundle" >> "$GITHUB_ENV" + cargo run --locked --release -p syntax-pack -- bundle --target "$TARGET" --out "$bundle" --jobs 4 + + - name: Build the binding + run: | + out="bindings/node/npm/$PLATFORM" + node bindings/node/scripts/build.mjs --locked --target "$TARGET" --out "$out" + cp bindings/node/LICENSE-MIT bindings/node/LICENSE-APACHE "$out/" + cp "target/syntax-bundle/$TARGET/THIRD_PARTY_NOTICES.md" "$out/" + ls -l "$out" + + # Fails on a crate license outside the script's allowlist. + - name: Write the Rust notices + run: | + node bindings/node/scripts/notices.mjs --locked --target "$TARGET" --out "bindings/node/npm/$PLATFORM" + grep -c '^## ' "bindings/node/npm/$PLATFORM/RUST_NOTICES.md" + + - name: Sign ad hoc (macOS) + if: runner.os == 'macOS' + run: codesign --force --sign - "bindings/node/npm/$PLATFORM/diffy.$PLATFORM.node" + + - name: Check the glibc it needs (Linux) + if: runner.os == 'Linux' + run: | + node="bindings/node/npm/$PLATFORM/diffy.$PLATFORM.node" + need=$(objdump -T "$node" | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' | sort -uV | tail -1) + echo "needs glibc $need" + if [ "$(printf '%s\n%s\n' "$need" "$MAX_GLIBC" | sort -V | tail -1)" != "$MAX_GLIBC" ]; then + echo "::error::$PLATFORM needs glibc $need, newer than $MAX_GLIBC" + exit 1 + fi + + - name: Test + working-directory: bindings/node + run: | + node -p "'node ' + process.version + ' ' + process.platform + '-' + process.arch" + DIFFY_NODE_BINARY="npm/$PLATFORM/diffy.$PLATFORM.node" npm test + + - uses: actions/upload-artifact@v4 + with: + name: node-${{ matrix.platform }} + path: bindings/node/npm/${{ matrix.platform }}/ + if-no-files-found: error + + publish: + needs: build + if: startsWith(github.ref, 'refs/tags/node-v') + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + # npm provenance. + id-token: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + + - uses: actions/download-artifact@v4 + with: + pattern: node-* + path: artifacts + + - name: Assemble the platform packages + run: | + for dir in bindings/node/npm/*/; do + platform=$(basename "$dir") + cp -R "artifacts/node-$platform/." "$dir" + test -f "$dir/diffy.$platform.node" + test -f "$dir/THIRD_PARTY_NOTICES.md" + test -f "$dir/RUST_NOTICES.md" + done + + - name: Check versions against the tag + env: + TAG: ${{ github.ref_name }} + run: | + node - <<'EOF' + const { readFileSync, readdirSync } = require('node:fs') + const version = process.env.TAG.replace(/^node-v/, '') + const read = (path) => JSON.parse(readFileSync(path, 'utf8')) + const main = read('bindings/node/package.json') + const problems = [] + if (main.version !== version) problems.push(`@diffy/node is ${main.version}`) + for (const platform of readdirSync('bindings/node/npm')) { + const pkg = read(`bindings/node/npm/${platform}/package.json`) + if (pkg.version !== version) problems.push(`${pkg.name} is ${pkg.version}`) + if (main.optionalDependencies[pkg.name] !== version) problems.push(`@diffy/node pins ${pkg.name}@${main.optionalDependencies[pkg.name]}`) + } + if (Object.keys(main.optionalDependencies).length !== readdirSync('bindings/node/npm').length) problems.push('optionalDependencies and npm/ differ') + if (problems.length) { + console.error(`tag ${process.env.TAG} wants ${version}:\n ${problems.join('\n ')}`) + process.exit(1) + } + EOF + + # Platform packages first, so @diffy/node never names a missing one. + # Published versions are skipped, so a failed run can be rerun. + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + publish() { + name=$(node -p "require('./$1/package.json').name") + version=$(node -p "require('./$1/package.json').version") + if npm view "$name@$version" version > /dev/null 2>&1; then + echo "$name@$version is already published" + else + npm publish "./$1" --provenance --access public + fi + } + for dir in bindings/node/npm/*; do publish "$dir"; done + publish bindings/node diff --git a/AGENTS.md b/AGENTS.md index 82ba3784..3fe6b521 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,10 +27,14 @@ and syntax code stays here. - `src/apprt/` — runtime and services split per stream; all blocking Git/fs/network/AI work runs on workers and reports through `transport.rs`. - `src/core/` — product logic: compare backends, VCS, review, syntax, themes. +- `crates/diffy-compare/` — headless file bytes to `DiffDocument` (semantic, + else line diff); no UI, Git or network deps. - `crates/diffy-diff/` — diff model; `crates/diffy-ui/` — diff view on Quark; `crates/diffy-syntax/` — tree-sitter packs (see its `PACKS.md`); `tools/syntax-pack/` — pack builder; `registry.toml` is generated by `syntax-pack registry`, never edited by hand. +- `bindings/node/` — `@diffy/node`, napi-rs binding over `diffy-compare` with + bundled grammars; build with the `node` profile. - `crates/diffy-semantic/` — structural diff behind the semantic renderer; `tools/semantic-eval/` measures it against history and fixtures. - `crates/difftastic/` is vendored and out-of-workspace, built only for diff --git a/Cargo.lock b/Cargo.lock index ae88fec9..40d8452c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,7 +305,7 @@ version = "0.38.0+1.3.281" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" dependencies = [ - "libloading", + "libloading 0.8.9", ] [[package]] @@ -808,6 +808,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -994,6 +1003,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" + [[package]] name = "cursor-icon" version = "1.2.0" @@ -1118,7 +1133,7 @@ dependencies = [ "ignore", "lazy_static", "libc", - "libloading", + "libloading 0.8.9", "line-numbers", "log", "owo-colors", @@ -1205,6 +1220,7 @@ dependencies = [ "arboard", "clap", "difftastic", + "diffy-compare", "diffy-diff", "diffy-semantic", "diffy-syntax", @@ -1243,6 +1259,19 @@ dependencies = [ "winit", ] +[[package]] +name = "diffy-compare" +version = "0.3.7" +dependencies = [ + "diffy-diff", + "diffy-semantic", + "diffy-syntax", + "gix-imara-diff", + "serde", + "serde_json", + "tracing", +] + [[package]] name = "diffy-diff" version = "0.3.7" @@ -1251,6 +1280,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "diffy-node" +version = "0.3.7" +dependencies = [ + "diffy-compare", + "diffy-syntax", + "napi", + "napi-build", + "napi-derive", + "serde_json", +] + [[package]] name = "diffy-semantic" version = "0.3.7" @@ -1264,7 +1305,7 @@ name = "diffy-syntax" version = "0.3.7" dependencies = [ "diffy-syntax", - "libloading", + "libloading 0.8.9", "quark-update", "ring", "serde", @@ -1361,7 +1402,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" dependencies = [ - "libloading", + "libloading 0.8.9", ] [[package]] @@ -1738,6 +1779,21 @@ dependencies = [ "libc", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1745,6 +1801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1753,6 +1810,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -1801,10 +1869,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -3510,7 +3581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading", + "libloading 0.8.9", "pkg-config", ] @@ -3603,6 +3674,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -3884,6 +3965,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "napi" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907ff7366fb7182e28edd0f08ef8ac5d6b13b8d65a3c0b3a10bf3ac092437d6b" +dependencies = [ + "bitflags 2.11.0", + "ctor", + "futures", + "libc", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash 2.1.2", +] + +[[package]] +name = "napi-build" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b899b545d3aa6dca985939059f258c5488d34e4ecf39c274e20009748f4b846d" + +[[package]] +name = "napi-derive" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32eb00f92f84cf462fc328f49082b29c2a7d2eebeb1d447d19c1f9fdd9f89ee" +dependencies = [ + "convert_case", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "napi-derive-backend" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fe28c64d8fc12e69f9b817d0febdbe7914555760f1e353106281b8ff99cc90" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "napi-sys" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d153beb7d5c973f4b9395be8a83054a28968b61188f1dd1a2cc2321af22daea2" +dependencies = [ + "libloading 0.9.0", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3949,6 +4086,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -5730,6 +5873,7 @@ version = "0.3.7" dependencies = [ "clap", "diffy", + "diffy-compare", "diffy-diff", "diffy-semantic", "diffy-syntax", @@ -6126,7 +6270,7 @@ checksum = "5dbb9f2928b6654ccc28d4ddfef5213e97ed66afed4907774d049b376c62a838" dependencies = [ "js-sys", "libc", - "libloading", + "libloading 0.8.9", "memfd", "memmap2", "serde", @@ -6227,7 +6371,7 @@ version = "0.3.7" dependencies = [ "cc", "diffy-syntax", - "libloading", + "libloading 0.8.9", "quark-update", "ring", "serde", @@ -7817,7 +7961,7 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading", + "libloading 0.8.9", "log", "metal", "naga", @@ -8292,7 +8436,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "libloading", + "libloading 0.8.9", "once_cell", "rustix 1.1.4", "x11rb-protocol", diff --git a/Cargo.toml b/Cargo.toml index 6baa9b03..758f4521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ ".", + "bindings/node", + "crates/diffy-compare", "crates/diffy-diff", "crates/diffy-semantic", "crates/diffy-syntax", @@ -94,6 +96,7 @@ parking_lot = "0.12" libc = "0.2" tempfile = "3.14" webbrowser = "1.0" +diffy-compare = { path = "crates/diffy-compare" } diffy-diff = { path = "crates/diffy-diff" } diffy-semantic = { path = "crates/diffy-semantic" } diffy-syntax = { path = "crates/diffy-syntax", features = ["download"] } @@ -135,6 +138,11 @@ codegen-units = 1 panic = "abort" strip = true +# bindings/node: a panic rejects the call instead of aborting the Node process. +[profile.node] +inherits = "release" +panic = "unwind" + [patch.crates-io] accesskit_unix = { git = "https://github.com/seatedro/quark", rev = "b0d4abf67fb59d9e9fe66fbda78bda62bad160b5" } cosmic-text = { git = "https://github.com/seatedro/quark", rev = "b0d4abf67fb59d9e9fe66fbda78bda62bad160b5" } diff --git a/README.md b/README.md index 5359b916..3299bbc1 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,6 @@ dx serve --hot-patch --features hot-reload ## License -GPL-3.0 +- The app: GPL-3.0. +- `diffy-diff`, `diffy-semantic`, `diffy-syntax`, `diffy-compare`, and the + Node binding: MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE` in each). diff --git a/bindings/node/.gitignore b/bindings/node/.gitignore new file mode 100644 index 00000000..09b9643e --- /dev/null +++ b/bindings/node/.gitignore @@ -0,0 +1,8 @@ +*.node +node_modules/ +# Copied into each platform package at publish time. +npm/*/LICENSE-* +npm/*/THIRD_PARTY_NOTICES.md +npm/*/RUST_NOTICES.md +# Written by `npm run notices`. +/RUST_NOTICES.md diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml new file mode 100644 index 00000000..5429ffff --- /dev/null +++ b/bindings/node/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "diffy-node" +description = "Node binding for diffy-compare, published as @diffy/node." +version.workspace = true +edition = "2024" +rust-version = "1.92" +license = "MIT OR Apache-2.0" +publish = false + +[lib] +crate-type = ["cdylib"] + +# Build with `--profile node`: release's panic = "abort" would kill the host. +[dependencies] +diffy-compare = { path = "../../crates/diffy-compare", features = ["serde"] } +# Grammars compiled in from $DIFFY_SYNTAX_BUNDLE; no packs, no network. +diffy-syntax = { path = "../../crates/diffy-syntax", features = ["bundled"] } +# `napi4`: lowest with thread-safe functions. `dyn-symbols`: Node-API is +# resolved from the host at load, so one file loads in Node and Electron and +# Windows needs no node.lib. +napi = { version = "3", default-features = false, features = ["napi4", "dyn-symbols"] } +# No `type-def`: index.d.ts is hand-written from FORMAT.md. +napi-derive = { version = "3", default-features = false, features = ["strict"] } +serde_json = "1.0" + +[build-dependencies] +# Addon link flags (macOS dynamic lookup). +napi-build = "2" diff --git a/bindings/node/LICENSE-APACHE b/bindings/node/LICENSE-APACHE new file mode 100644 index 00000000..9267b22b --- /dev/null +++ b/bindings/node/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Rohit (seatedro) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bindings/node/LICENSE-MIT b/bindings/node/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/bindings/node/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/node/README.md b/bindings/node/README.md new file mode 100644 index 00000000..407ebd9d --- /dev/null +++ b/bindings/node/README.md @@ -0,0 +1,64 @@ +# @diffy/node + +Semantic file diffs for Node. Pass file contents, get back a JSON document +to render. The format is in +[FORMAT.md](https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md); +`index.d.ts` types it. + +```js +const { diff, expand } = require('@diffy/node') + +const doc = await diff( + { files: [{ status: 'modified', old_path: 'a.rs', new_path: 'a.rs', old, new: next }], syntax: true, budget_ms: 2000 }, + { signal }, +) +const rows = await expand(file, doc.files[0].gap_after, { syntax: true }) +``` + +## API + +- `diff(request, { signal })` → `Document`. Contents are bytes or UTF-8 strings. + - `budget_ms`: whole request, from the call. Late files get line diffs (`request_budget`) and no syntax. + - `file_budget_ms`: per-file semantic budget, default 400. + - `syntax`, `syntax_max_bytes` (default 1 MiB): syntax spans. +- `diffJson(request, { signal })` → the same document as a JSON string, for forwarding without parsing. +- `expand(file, gap, { syntax })` → context rows of a gap or part of one. +- `configure({ max_concurrent_requests })`: concurrent `diff` calls, default cores / 4. Extra calls queue in order. +- `languages()`: grammars compiled in. Others get line diffs. +- `FORMAT_VERSION`. + +An aborted `signal` rejects with `AbortError`. Bad input rejects with a +`TypeError` whose `code` is `"InvalidArg"`. + +## Notes + +- Work runs on its own threads, never libuv's pool. `expand` skips the queue. +- No network or disk access: grammars are compiled in. +- Renames are detected but not checked against the rest of the repository. +- Node-API 4 only: one binary works in Node 18+ and Electron (unpack `*.node` from asar). +- A Rust panic rejects the call; a crash in a grammar's C code kills the process. + +## Packages + +| Package | Platform | +| --- | --- | +| `@diffy/node-darwin-arm64` | macOS 11+, Apple silicon | +| `@diffy/node-darwin-x64` | macOS 11+, Intel | +| `@diffy/node-linux-x64-gnu` | Linux x64, glibc 2.28+ | +| `@diffy/node-linux-arm64-gnu` | Linux arm64, glibc 2.28+ | +| `@diffy/node-win32-x64-msvc` | Windows x64 | + +Each ships `THIRD_PARTY_NOTICES.md` (grammars) and `RUST_NOTICES.md` (Rust crates). + +## Development + +```sh +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" rust typescript tsx +cd bindings/node +DIFFY_SYNTAX_BUNDLE="$PWD/../../target/syntax-bundle" npm run build +npm test +``` + +- `DIFFY_NODE_BINARY=path/to/diffy.node` loads a specific binary. +- `npm run notices -- --target TRIPLE --out DIR` writes `RUST_NOTICES.md`. +- Release: set the version in `package.json`, `npm/*/package.json` and `optionalDependencies`, then push a `node-v` tag. diff --git a/bindings/node/binding.js b/bindings/node/binding.js new file mode 100644 index 00000000..3e382f7d --- /dev/null +++ b/bindings/node/binding.js @@ -0,0 +1,40 @@ +'use strict' + +// Loads $DIFFY_NODE_BINARY (resolved from the cwd), else a local +// `npm run build` output, else this platform's package. + +const { existsSync, readFileSync } = require('node:fs') +const { join, resolve } = require('node:path') + +function platformName() { + const { platform, arch } = process + if (platform === 'linux') return `linux-${arch}-${isMusl() ? 'musl' : 'gnu'}` + if (platform === 'win32') return `win32-${arch}-msvc` + return `${platform}-${arch}` +} + +function isMusl() { + try { + return readFileSync('/usr/bin/ldd', 'utf8').includes('musl') + } catch { + return false + } +} + +function load() { + if (process.env.DIFFY_NODE_BINARY) return require(resolve(process.env.DIFFY_NODE_BINARY)) + const name = platformName() + const local = join(__dirname, `diffy.${name}.node`) + if (existsSync(local)) return require(local) + try { + return require(`@diffy/node-${name}`) + } catch (error) { + throw new Error( + `@diffy/node has no binary for ${name}: install @diffy/node-${name} ` + + '(npm skips optional dependencies with --omit=optional), or set DIFFY_NODE_BINARY', + { cause: error }, + ) + } +} + +module.exports = load() diff --git a/bindings/node/build.rs b/bindings/node/build.rs new file mode 100644 index 00000000..0f1b0100 --- /dev/null +++ b/bindings/node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts new file mode 100644 index 00000000..ecdc33a1 --- /dev/null +++ b/bindings/node/index.d.ts @@ -0,0 +1,279 @@ +// Output follows Diffy's display format, version 1, specified in +// https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md +// Ignore unknown fields: new optional fields keep FORMAT_VERSION. + +/** Display format version of every Document this build returns. */ +export const FORMAT_VERSION: number + +/** Canonical names of the bundled grammars, e.g. `"rust"`. */ +export function languages(): string[] + +/** + * Diffs off the JS thread; one File per input, in order. Waits in a + * process-wide FIFO while `max_concurrent_requests` others run. + * Rejects with an `AbortError` (`code: "ABORT_ERR"`, `cause`: the signal's + * reason) on abort, and a `TypeError` (`code: "InvalidArg"`) for bad input. + */ +export function diff(request: DiffRequest, options?: DiffOptions): Promise + +/** `diff`, unparsed: forward the Document's JSON without parsing it on the event loop. */ +export function diffJson(request: DiffRequest, options?: DiffOptions): Promise + +/** + * Context rows of `gap` (a Document's Gap or a sub-range of one); `file` must + * hold the contents the Document was made from. Fewer than `gap.len` rows past + * either side's end or where the sides differ. Bad input rejects as in `diff`. + */ +export function expand(file: FileInput, gap: Gap, options?: ExpandOptions): Promise + +/** Sets the given options process-wide; returns all. Throws a `TypeError` (`code: "InvalidArg"`) for a bad value. */ +export function configure(options?: Configuration): Required + +export interface Configuration { + /** + * Concurrent `diff`/`diffJson` calls; the rest queue. Positive integer. + * Default: cores / 4, at least 1. Raising it starts queued calls at once; + * lowering it lets running ones finish. `expand` is never queued. + */ + max_concurrent_requests?: number +} + +// ---- Input ------------------------------------------------------------- + +export type Status = 'modified' | 'added' | 'deleted' | 'renamed' | 'copied' + +/** + * Contents are bytes; a string means its UTF-8. Missing contents are empty. + * Needs a path; the language comes from `new_path ?? old_path`. + */ +export interface FileInput { + status: Status + /** Absent for an added file. */ + old_path?: string + /** Absent for a deleted file. */ + new_path?: string + old?: Uint8Array | string + new?: Uint8Array | string +} + +export interface DiffRequest { + files: FileInput[] + /** + * Whole-request budget from the call, time queued included. Files started + * after it get line diffs (`fallback.kind: "request_budget"`); with + * `syntax`, files projected after it get no syntax spans or scope headers. + */ + budget_ms?: number + /** Per-file semantic diff budget. Default 400. */ + file_budget_ms?: number + /** Add syntax spans to rows. Default false. */ + syntax?: boolean + /** Sides longer than this, in bytes, get no syntax spans. Default 1 MiB. */ + syntax_max_bytes?: number +} + +export interface DiffOptions { + signal?: AbortSignal +} + +export interface ExpandOptions { + /** Add syntax spans to rows. Default false. */ + syntax?: boolean +} + +// ---- Output: the display format, version 1 ------------------------------ +// +// Line numbers are 1-based; ranges are half-open; span offsets are UTF-16 +// code units within the row's `text`, so `text.slice(start, end)` is the +// span. Absent arrays mean `[]`, absent booleans `false`. + +export interface Document { + version: 1 + /** In request order. */ + files: File[] +} + +export interface File { + status: Status + /** Absent for an added file. */ + old_path?: string + /** Absent for a deleted file. Show `new_path ?? old_path`. */ + new_path?: string + /** + * The extension (`rs`) or, without one, the lowercased name (`dockerfile`). + * Absent when the path names none or `fallback.kind` is `no_grammar`. + */ + language?: string + /** A binary file has no `hunks` and no `gap_after`. */ + binary?: boolean + additions: number + deletions: number + /** The old file's last line has no newline. */ + old_missing_newline?: boolean + /** The new file's last line has no newline. */ + new_missing_newline?: boolean + /** Present when the file got a line diff instead of a semantic one. */ + fallback?: Fallback + /** In file order. Render each hunk's `gap_before`, header, rows; then `gap_after`. */ + hunks?: Hunk[] + /** Unchanged lines after the last hunk; the whole file when there are no hunks. */ + gap_after?: Gap + /** Items the semantic diff found changed: new-side items, then old-side ones. */ + outline?: OutlineItem[] + /** Code the semantic diff found moved. */ + moves?: Move[] + /** Identifiers the semantic diff found renamed. */ + renames?: Rename[] +} + +export interface Fallback { + /** + * `no_grammar`: no language for the path. `grammar_unavailable`: its grammar + * failed to load. `semantic`: the semantic diff gave up, see `reason`. + * `request_budget`: the request deadline passed first. + */ + kind: 'no_grammar' | 'grammar_unavailable' | 'semantic' | 'request_budget' + /** Present when `kind` is `semantic`. */ + reason?: 'too_large' | 'parse_errors' | 'error_coverage' | 'unparsed' | 'budget' | 'timeout' + /** For people; wording may change. Branch on `kind` and `reason`. */ + message: string +} + +export interface Hunk { + /** A unified diff's `@@ -old_start,old_len +new_start,new_len @@`. */ + old_start: number + old_len: number + new_start: number + new_len: number + /** Item enclosing the first change, for the header line. */ + header?: string + /** Unchanged lines since the previous hunk or the top of the file. */ + gap_before?: Gap + /** In display order; each side's line numbers ascend. */ + rows: Row[] +} + +/** Unchanged lines left out of the hunks; identical on both sides. */ +export interface Gap { + old_start: number + new_start: number + /** At least 1. */ + len: number +} + +export type Row = ContextRow | RemovedRow | AddedRow | ModifiedRow + +/** An unchanged line. */ +export interface ContextRow { + kind: 'context' + old_line: number + new_line: number + text: string + /** From the new file. */ + syntax?: SyntaxSpan[] +} + +export interface RemovedRow { + kind: 'removed' + old: Line +} + +export interface AddedRow { + kind: 'added' + new: Line +} + +/** A removed line beside the added line it became. */ +export interface ModifiedRow { + kind: 'modified' + old: Line + new: Line +} + +/** One side of a changed row. */ +export interface Line { + /** Line number on its side. */ + line: number + /** No terminator, no trailing `\r`; invalid UTF-8 shown as U+FFFD. */ + text: string + /** What changed within the line. Sorted, never overlapping. */ + changes?: ChangeSpan[] + /** From this side's file. */ + syntax?: SyntaxSpan[] +} + +export interface ChangeSpan { + /** UTF-16 offset within the line's `text`. */ + start: number + end: number + /** + * `novel`: syntax only on this side; `novel_word`: a changed word in an + * otherwise matched line; `unchanged_context`: unchanged syntax inside a + * changed line; `renamed`: an identifier matched as renamed. + */ + intensity: 'novel' | 'novel_word' | 'unchanged_context' | 'renamed' +} + +export type SyntaxClass = + | 'keyword' + | 'string' + | 'comment' + | 'number' + | 'type' + | 'function' + | 'operator' + | 'punctuation' + | 'variable' + | 'constant' + | 'builtin' + | 'attribute' + | 'tag' + | 'property' + | 'namespace' + | 'label' + | 'preprocessor' + +export interface SyntaxSpan { + /** UTF-16 offset within the row's `text`. Sorted, never overlapping. */ + start: number + end: number + /** May gain values without a version bump; treat unknown ones as plain text. */ + class: SyntaxClass | (string & {}) +} + +export interface OutlineItem { + /** Which file `line` refers to: `removed` items are old, the rest new. */ + side: 'old' | 'new' + /** The item's first line. */ + line: number + /** The item's last line, inclusive. */ + end_line?: number + /** The grammar's node kind, e.g. `function_item`. */ + kind: string + name: string + /** `renamed`: only identifiers inside it were renamed. */ + change: 'added' | 'removed' | 'renamed' | 'moved' | 'edited' + /** Changed lines of an `edited` item. */ + edits?: number +} + +/** `old_len` lines from old line `old_start` moved to `new_len` lines from new line `new_start`. */ +export interface Move { + old_start: number + old_len: number + new_start: number + new_len: number + /** Name of the item the code left. */ + from?: string + /** Name of the item the code arrived in. */ + into?: string + /** Name of the moved item itself, when the code is one. */ + item?: string +} + +export interface Rename { + old: string + new: string + /** Occurrences renamed. */ + uses: number +} diff --git a/bindings/node/index.js b/bindings/node/index.js new file mode 100644 index 00000000..384d943f --- /dev/null +++ b/bindings/node/index.js @@ -0,0 +1,84 @@ +'use strict' + +const native = require('./binding.js') + +const FORMAT_VERSION = native.FORMAT_VERSION + +function languages() { + return native.languages() +} + +async function diff(request, options) { + return JSON.parse(await diffJson(request, options)) +} + +// Async, so argument errors reject like every other failure. +async function diffJson(request, options) { + const signal = options?.signal + if (typeof request !== 'object' || request === null) throw invalid('request must be an object') + if (signal !== undefined && typeof signal?.addEventListener !== 'function') { + throw invalid('options.signal must be an AbortSignal') + } + if (signal?.aborted) throw abortError(signal) + const token = new native.CancelToken() + const work = call(() => native.diff(request, token)) + if (signal === undefined) return work + + // Abort rejects at once and cancels the native request; its own + // rejection is dropped. + let onAbort + const aborted = new Promise((_, reject) => { + onAbort = () => { + token.cancel() + reject(abortError(signal)) + } + signal.addEventListener('abort', onAbort, { once: true }) + }) + work.catch(() => {}) + try { + return await Promise.race([work, aborted]) + } catch (error) { + if (error?.code === 'Cancelled') throw abortError(signal) + throw error + } finally { + signal.removeEventListener('abort', onAbort) + } +} + +async function expand(file, gap, options) { + return JSON.parse(await call(() => native.expand(file, gap, options?.syntax))) +} + +function configure(options) { + if (options !== undefined && (typeof options !== 'object' || options === null)) { + throw invalid('options must be an object') + } + const state = call(() => native.queue(options?.max_concurrent_requests)) + return { max_concurrent_requests: state.limit } +} + +// Sync throws are argument errors (codes vary by conversion), except +// GenericFailure: no thread could start. +function call(binding) { + try { + return binding() + } catch (error) { + if (error?.code === 'GenericFailure') throw error + throw invalid(error?.message ?? String(error), error) + } +} + +function invalid(message, cause) { + const error = new TypeError(message, { cause }) + error.code = 'InvalidArg' + return error +} + +function abortError(signal) { + const error = new Error('The operation was aborted', { cause: signal.reason }) + error.name = 'AbortError' + error.code = 'ABORT_ERR' + return error +} + +module.exports = { FORMAT_VERSION, languages, configure, diff, diffJson, expand } diff --git a/bindings/node/notices/napi-rs-LICENSE b/bindings/node/notices/napi-rs-LICENSE new file mode 100644 index 00000000..7fe7e35e --- /dev/null +++ b/bindings/node/notices/napi-rs-LICENSE @@ -0,0 +1,43 @@ +MIT License + +Copyright (c) 2020-present LongYinan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +MIT License + +Copyright (c) 2018 GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/node/npm/darwin-arm64/README.md b/bindings/node/npm/darwin-arm64/README.md new file mode 100644 index 00000000..f3e182c3 --- /dev/null +++ b/bindings/node/npm/darwin-arm64/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-darwin-arm64` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Apple silicon. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/darwin-arm64/package.json b/bindings/node/npm/darwin-arm64/package.json new file mode 100644 index 00000000..6642c374 --- /dev/null +++ b/bindings/node/npm/darwin-arm64/package.json @@ -0,0 +1,32 @@ +{ + "name": "@diffy/node-darwin-arm64", + "version": "0.1.0", + "description": "The @diffy/node binary for macOS on Apple silicon.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/darwin-arm64" + }, + "main": "diffy.darwin-arm64.node", + "files": [ + "diffy.darwin-arm64.node", + "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/darwin-x64/README.md b/bindings/node/npm/darwin-x64/README.md new file mode 100644 index 00000000..e16a890c --- /dev/null +++ b/bindings/node/npm/darwin-x64/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-darwin-x64` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Intel. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/darwin-x64/package.json b/bindings/node/npm/darwin-x64/package.json new file mode 100644 index 00000000..1bc8415b --- /dev/null +++ b/bindings/node/npm/darwin-x64/package.json @@ -0,0 +1,32 @@ +{ + "name": "@diffy/node-darwin-x64", + "version": "0.1.0", + "description": "The @diffy/node binary for macOS on Intel.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/darwin-x64" + }, + "main": "diffy.darwin-x64.node", + "files": [ + "diffy.darwin-x64.node", + "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/linux-arm64-gnu/README.md b/bindings/node/npm/linux-arm64-gnu/README.md new file mode 100644 index 00000000..7a4374f8 --- /dev/null +++ b/bindings/node/npm/linux-arm64-gnu/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-linux-arm64-gnu` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux arm64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/linux-arm64-gnu/package.json b/bindings/node/npm/linux-arm64-gnu/package.json new file mode 100644 index 00000000..adaf1043 --- /dev/null +++ b/bindings/node/npm/linux-arm64-gnu/package.json @@ -0,0 +1,35 @@ +{ + "name": "@diffy/node-linux-arm64-gnu", + "version": "0.1.0", + "description": "The @diffy/node binary for Linux arm64 with glibc 2.28 or newer.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/linux-arm64-gnu" + }, + "main": "diffy.linux-arm64-gnu.node", + "files": [ + "diffy.linux-arm64-gnu.node", + "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/linux-x64-gnu/README.md b/bindings/node/npm/linux-x64-gnu/README.md new file mode 100644 index 00000000..ce6d7c0e --- /dev/null +++ b/bindings/node/npm/linux-x64-gnu/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-linux-x64-gnu` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux x64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/linux-x64-gnu/package.json b/bindings/node/npm/linux-x64-gnu/package.json new file mode 100644 index 00000000..35d1d24f --- /dev/null +++ b/bindings/node/npm/linux-x64-gnu/package.json @@ -0,0 +1,35 @@ +{ + "name": "@diffy/node-linux-x64-gnu", + "version": "0.1.0", + "description": "The @diffy/node binary for Linux x64 with glibc 2.28 or newer.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/linux-x64-gnu" + }, + "main": "diffy.linux-x64-gnu.node", + "files": [ + "diffy.linux-x64-gnu.node", + "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/win32-x64-msvc/README.md b/bindings/node/npm/win32-x64-msvc/README.md new file mode 100644 index 00000000..8adaeb79 --- /dev/null +++ b/bindings/node/npm/win32-x64-msvc/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-win32-x64-msvc` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Windows x64. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/win32-x64-msvc/package.json b/bindings/node/npm/win32-x64-msvc/package.json new file mode 100644 index 00000000..a9107c0c --- /dev/null +++ b/bindings/node/npm/win32-x64-msvc/package.json @@ -0,0 +1,32 @@ +{ + "name": "@diffy/node-win32-x64-msvc", + "version": "0.1.0", + "description": "The @diffy/node binary for Windows x64.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/win32-x64-msvc" + }, + "main": "diffy.win32-x64-msvc.node", + "files": [ + "diffy.win32-x64-msvc.node", + "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/package.json b/bindings/node/package.json new file mode 100644 index 00000000..59712482 --- /dev/null +++ b/bindings/node/package.json @@ -0,0 +1,48 @@ +{ + "name": "@diffy/node", + "version": "0.1.0", + "description": "Diffy's semantic file diff for Node: hand it file contents, get back a JSON diff to render.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node" + }, + "homepage": "https://github.com/seatedro/diffy/tree/master/bindings/node", + "keywords": ["diff", "semantic-diff", "tree-sitter", "git"], + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "index.js", + "index.d.ts", + "binding.js", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "engines": { + "node": ">=18" + }, + "scripts": { + "build": "node scripts/build.mjs", + "notices": "node scripts/notices.mjs", + "test": "node --test test/diffy.test.mjs" + }, + "optionalDependencies": { + "@diffy/node-darwin-arm64": "0.1.0", + "@diffy/node-darwin-x64": "0.1.0", + "@diffy/node-linux-x64-gnu": "0.1.0", + "@diffy/node-linux-arm64-gnu": "0.1.0", + "@diffy/node-win32-x64-msvc": "0.1.0" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/scripts/build.mjs b/bindings/node/scripts/build.mjs new file mode 100644 index 00000000..0ddc2043 --- /dev/null +++ b/bindings/node/scripts/build.mjs @@ -0,0 +1,74 @@ +// Builds the addon as `diffy..node`, the name binding.js and the +// platform packages load. +// +// node scripts/build.mjs [--target TRIPLE] [--out DIR] [--profile NAME] [--locked] +// +// DIFFY_SYNTAX_BUNDLE: absolute `syntax-pack bundle --out` dir; unset, no grammars. + +import { spawnSync } from 'node:child_process' +import { copyFileSync, mkdirSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' + +const PLATFORMS = { + 'aarch64-apple-darwin': 'darwin-arm64', + 'x86_64-apple-darwin': 'darwin-x64', + 'x86_64-unknown-linux-gnu': 'linux-x64-gnu', + 'aarch64-unknown-linux-gnu': 'linux-arm64-gnu', + 'x86_64-pc-windows-msvc': 'win32-x64-msvc', +} + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const { values } = parseArgs({ + options: { + target: { type: 'string' }, + out: { type: 'string', default: root }, + profile: { type: 'string', default: 'node' }, + locked: { type: 'boolean', default: false }, + }, +}) + +const target = values.target ?? hostTriple() +const platform = PLATFORMS[target] +if (!platform) fail(`no npm platform for ${target}; known: ${Object.keys(PLATFORMS).join(', ')}`) +if (!process.env.DIFFY_SYNTAX_BUNDLE) console.warn('DIFFY_SYNTAX_BUNDLE is not set: no grammars') + +const args = ['build', '-p', 'diffy-node', '--profile', values.profile, '--message-format=json-render-diagnostics'] +if (values.target) args.push('--target', values.target) +if (values.locked) args.push('--locked') +const cargo = spawnSync('cargo', args, { + cwd: root, + stdio: ['ignore', 'pipe', 'inherit'], + encoding: 'utf8', + maxBuffer: 1 << 30, +}) +if (cargo.error) fail(`cargo: ${cargo.error.message}`) +if (cargo.status !== 0) fail(`cargo build failed (${cargo.status})`) + +let library +for (const line of cargo.stdout.split('\n')) { + if (!line.startsWith('{')) continue + const message = JSON.parse(line) + if (message.reason === 'compiler-artifact' && message.target?.name === 'diffy_node') { + library = message.filenames.find((file) => /\.(so|dylib|dll)$/.test(file)) ?? library + } +} +if (!library) fail('cargo reported no diffy_node library') + +mkdirSync(values.out, { recursive: true }) +const output = join(values.out, `diffy.${platform}.node`) +copyFileSync(library, output) +console.log(output) + +function hostTriple() { + const rustc = spawnSync('rustc', ['-vV'], { encoding: 'utf8' }) + const host = rustc.stdout?.match(/^host: (\S+)$/m)?.[1] + if (!host) fail('could not read the host triple from `rustc -vV`') + return host +} + +function fail(message) { + console.error(`build: ${message}`) + process.exit(1) +} diff --git a/bindings/node/scripts/notices.mjs b/bindings/node/scripts/notices.mjs new file mode 100644 index 00000000..51d8f4c0 --- /dev/null +++ b/bindings/node/scripts/notices.mjs @@ -0,0 +1,186 @@ +// Writes RUST_NOTICES.md: each Rust crate linked into the addon for a target, +// with its license files. Fails, writing nothing, on a license outside the +// allowlist. Grammars are credited in the bundle's THIRD_PARTY_NOTICES.md. +// +// node scripts/notices.mjs [--target TRIPLE] [--out DIR] [--locked] + +import { spawnSync } from 'node:child_process' +import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' + +const PACKAGE = 'diffy-node' +// Licenses this notice satisfies. MPL-2.0 binds only the crate's own files, +// shipped unmodified. +const PERMITTED = new Set([ + 'MIT', + 'Apache-2.0', + 'BSD-2-Clause', + 'BSD-3-Clause', + 'ISC', + 'Unicode-3.0', + 'Zlib', + '0BSD', + 'CC0-1.0', + 'BSL-1.0', + 'MPL-2.0', +]) +const PERMITTED_EXCEPTIONS = new Set(['LLVM-exception']) +const LICENSE_FILE = /^(licen[cs]e|copying|notice)/i +// napi-rs crates ship no license file; reproduce their repository's. +const NAPI_RS = { path: resolve(dirname(fileURLToPath(import.meta.url)), '../notices/napi-rs-LICENSE'), source: 'github.com/napi-rs/napi-rs' } +const FALLBACK_LICENSES = new Map(['napi', 'napi-sys', 'napi-derive', 'napi-derive-backend'].map((name) => [name, NAPI_RS])) + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const { values } = parseArgs({ + options: { + target: { type: 'string' }, + out: { type: 'string', default: root }, + locked: { type: 'boolean', default: false }, + }, +}) +const target = values.target ?? hostTriple() +const locked = values.locked ? ['--locked'] : [] + +// `cargo tree -p` resolves features for this package alone, as `cargo build +// -p` does; `cargo metadata` resolves the whole workspace and would add +// app-only crates (HTTP, TLS). +const tree = cargo(['tree', '-p', PACKAGE, '-e', 'normal', '--target', target, '--prefix', 'none', '--format', '{p}', ...locked]) +const used = new Set() +for (const line of tree.split('\n')) { + const match = line.match(/^(\S+) v(\S+)/) + if (match) used.add(`${match[1]} ${match[2]}`) +} +if (used.size === 0) fail(`cargo tree listed no crates for ${PACKAGE}`) + +const metadata = JSON.parse( + cargo(['metadata', '--format-version', '1', '--filter-platform', target, ...locked]), +) +const workspace = new Set(metadata.workspace_members) +const packages = new Map() +for (const pkg of metadata.packages) { + const key = `${pkg.name} ${pkg.version}` + if (!used.has(key)) continue + if (packages.has(key)) fail(`two packages are ${key}; cannot tell which is linked`) + packages.set(key, pkg) +} +for (const key of used) if (!packages.has(key)) fail(`cargo metadata has no package ${key}`) + +const crates = [...packages.values()].sort((a, b) => a.name.localeCompare(b.name) || a.version.localeCompare(b.version)) +const refused = crates.filter((pkg) => !permitted(pkg.license ?? '')) +if (refused.length > 0) { + for (const pkg of refused) { + console.error(`notices: ${pkg.name} ${pkg.version} is ${pkg.license ?? `unlicensed (license-file ${pkg.license_file})`}`) + } + fail(`${refused.length} crate(s) have a license outside the allowlist`) +} + +const ours = crates.filter((pkg) => workspace.has(pkg.id)) +const theirs = crates.filter((pkg) => !workspace.has(pkg.id)) +const out = [ + '# Rust crates in this binary', + '', + `The Rust crates compiled into \`@diffy/node\` for \`${target}\`, with their licenses and the license files each one ships. The tree-sitter grammars compiled in are listed in THIRD_PARTY_NOTICES.md.`, + '', + `Diffy's own crates (${ours.map((pkg) => `\`${pkg.name}\``).join(', ')}) are ${[...new Set(ours.map((pkg) => pkg.license))].join('; ')}: see LICENSE-MIT and LICENSE-APACHE.`, +] +// Identical license texts are printed once. +const printed = new Map() +for (const pkg of theirs) { + out.push('', `## ${pkg.name} ${pkg.version}`, '', `License: ${pkg.license}`) + if (pkg.repository) out.push(`Repository: ${pkg.repository}`) + const files = licenseFiles(pkg) + const fallback = FALLBACK_LICENSES.get(pkg.name) + if (files.length === 0 && !fallback) out.push('', `No license file in the package; its license is \`${pkg.license}\`.`) + const paths = files.map((file) => [file, resolve(dirname(pkg.manifest_path), file)]) + if (files.length === 0 && fallback) paths.push([`LICENSE (from ${fallback.source})`, fallback.path]) + for (const [file, path] of paths) { + const text = readFileSync(path, 'utf8') + const key = text.replaceAll('\r\n', '\n').replace(/[ \t]+$/gm, '').trim() + const first = printed.get(key) + if (first) { + out.push('', `${file}: the same text as ${first}.`) + continue + } + printed.set(key, `${pkg.name} ${pkg.version}'s ${file}`) + out.push('', `### ${file}`, '', ...fenced(key)) + } +} + +mkdirSync(values.out, { recursive: true }) +const output = join(values.out, 'RUST_NOTICES.md') +writeFileSync(output, `${out.join('\n')}\n`) +console.log(`${output}: ${theirs.length} crates`) + +/** The top-level license files of `pkg`, and its `license-file`. */ +function licenseFiles(pkg) { + const dir = dirname(pkg.manifest_path) + const files = new Set(readdirSync(dir).filter((name) => LICENSE_FILE.test(name) && statSync(join(dir, name)).isFile())) + if (pkg.license_file) files.add(pkg.license_file) + return [...files].sort() +} + +/** Whether an SPDX expression (or the legacy `A/B`) lets us ship the crate. */ +function permitted(expression) { + const tokens = expression.replaceAll('/', ' OR ').match(/[()]|[^\s()]+/g) ?? [] + let at = 0 + // Each side is always parsed, so `at` moves past it. + const any = () => { + let ok = all() + while (tokens[at] === 'OR') { + at++ + ok = all() || ok + } + return ok + } + const all = () => { + let ok = one() + while (tokens[at] === 'AND') { + at++ + ok = one() && ok + } + return ok + } + const one = () => { + const token = tokens[at++] + if (token === '(') { + const ok = any() + return tokens[at++] === ')' && ok + } + let ok = PERMITTED.has(token) + if (tokens[at] === 'WITH') { + at++ + ok = PERMITTED_EXCEPTIONS.has(tokens[at++]) && ok + } + return ok + } + const ok = any() + return ok && at === tokens.length +} + +/** `text` in a code fence longer than any backtick run inside it. */ +function fenced(text) { + const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length)) + const fence = '`'.repeat(Math.max(3, longest + 1)) + return [`${fence}text`, text, fence] +} + +function cargo(args) { + const run = spawnSync('cargo', args, { cwd: root, encoding: 'utf8', maxBuffer: 1 << 30, stdio: ['ignore', 'pipe', 'inherit'] }) + if (run.error) fail(`cargo: ${run.error.message}`) + if (run.status !== 0) fail(`cargo ${args[0]} failed (${run.status})`) + return run.stdout +} + +function hostTriple() { + const rustc = spawnSync('rustc', ['-vV'], { encoding: 'utf8' }) + const host = rustc.stdout?.match(/^host: (\S+)$/m)?.[1] + if (!host) fail('could not read the host triple from `rustc -vV`') + return host +} + +function fail(message) { + console.error(`notices: ${message}`) + process.exit(1) +} diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs new file mode 100644 index 00000000..0e9e846d --- /dev/null +++ b/bindings/node/src/lib.rs @@ -0,0 +1,292 @@ +//! Native half of `@diffy/node`; `index.js` is the API. +//! +//! Input is copied on the JS thread; work runs on threads of its own, not +//! libuv's pool (shared with fs, DNS and zlib), and settles through a +//! thread-safe function. `diff` waits in [`queue`]; `expand` starts at once. +//! Results cross as one JSON string. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use diffy_compare::wire::{self, WireOptions}; +use diffy_compare::{FileStatus, GrammarStore, Options}; +use diffy_syntax::StoreConfig; +use napi::bindgen_prelude::{Either, Object, Uint8Array}; +use napi::{Env, Error, Result, Status}; +use napi_derive::napi; + +mod queue; + +#[napi] +pub const FORMAT_VERSION: u32 = wire::FORMAT_VERSION; + +/// Bundled grammars, by canonical name. +#[napi] +pub fn languages() -> Vec { + diffy_syntax::bundled::languages() + .map(str::to_owned) + .collect() +} + +/// Built lazily off the JS thread; bundled only, so no network or file system. +fn grammars() -> &'static GrammarStore { + static STORE: OnceLock = OnceLock::new(); + STORE.get_or_init(|| GrammarStore::new(StoreConfig::new().bundled())) +} + +#[napi(object, object_to_js = false)] +pub struct FileInput { + pub status: String, + #[napi(js_name = "old_path")] + pub old_path: Option, + #[napi(js_name = "new_path")] + pub new_path: Option, + pub old: Option>, + pub new: Option>, +} + +#[napi(object, object_to_js = false)] +pub struct DiffRequest { + pub files: Vec, + #[napi(js_name = "budget_ms")] + pub budget_ms: Option, + #[napi(js_name = "file_budget_ms")] + pub file_budget_ms: Option, + pub syntax: Option, + #[napi(js_name = "syntax_max_bytes")] + pub syntax_max_bytes: Option, +} + +#[napi(object, object_to_js = false)] +pub struct Gap { + #[napi(js_name = "old_start")] + pub old_start: f64, + #[napi(js_name = "new_start")] + pub new_start: f64, + pub len: f64, +} + +/// Set by `index.js` from an `AbortSignal`. +#[napi] +pub struct CancelToken { + flag: Arc, +} + +#[napi] +impl CancelToken { + #[napi(constructor)] + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + flag: Arc::new(AtomicBool::new(false)), + } + } + + /// A queued request is rejected at once and never starts; a running one + /// stops at its next file. + #[napi] + pub fn cancel(&self) { + self.flag.store(true, Ordering::Relaxed); + queue::cancel(&self.flag); + } +} + +#[napi(object, object_from_js = false)] +pub struct QueueState { + pub limit: u32, + pub running: u32, + pub queued: u32, +} + +/// Sets the `diff` concurrency limit, if given, and reports the queue. +#[napi(js_name = "queue")] +pub fn queue_state(limit: Option) -> Result { + if let Some(limit) = limit { + if !(limit.is_finite() && limit >= 1.0 && limit.fract() == 0.0) { + return Err(invalid(format!( + "max_concurrent_requests: {limit} is not a positive integer" + ))); + } + // `as` saturates; usize::MAX means no limit anyway. + queue::set_limit(limit as usize); + } + let state = queue::state(); + let count = |n: usize| u32::try_from(n).unwrap_or(u32::MAX); + Ok(QueueState { + limit: count(state.limit), + running: count(state.running), + queued: count(state.queued), + }) +} + +/// Queued; resolves to the Document's JSON. +#[napi] +pub fn diff<'env>( + env: &'env Env, + request: DiffRequest, + cancel: &CancelToken, +) -> Result> { + // The request budget runs from the call, so it counts time queued. + let start = Instant::now(); + let deadline = request + .budget_ms + .map(|ms| duration("budget_ms", ms).map(|budget| start + budget)) + .transpose()?; + let file_budget = match request.file_budget_ms { + Some(ms) => duration("file_budget_ms", ms)?, + None => diffy_compare::FILE_BUDGET, + }; + let wire = wire_settings(request.syntax, request.syntax_max_bytes)?; + let inputs = request + .files + .into_iter() + .map(file_input) + .collect::>>()?; + let flag = Arc::clone(&cancel.flag); + let (deferred, promise) = env.create_deferred()?; + let work: queue::Work = Box::new(move || { + let grammars = grammars(); + let mut options = Options::new(grammars); + options.file_budget = file_budget; + options.deadline = deadline; + options.cancel = Some(&flag); + // No `vet`: the binding never sees the repository to confirm renames. + let diffs = diffy_compare::diff_files(&inputs, &options).map_err(compare_error)?; + let document = wire::document( + &inputs, + &diffs, + &WireOptions { + deadline, + cancel: Some(&flag), + ..wire.options(grammars) + }, + ); + serde_json::to_string(&document).map_err(json_error) + }); + queue::submit(queue::Job { + deferred, + cancel: Arc::clone(&cancel.flag), + work, + }); + Ok(promise) +} + +/// Context rows of `gap` as JSON. Not queued: one file, and a user is waiting. +#[napi] +pub fn expand<'env>( + env: &'env Env, + file: FileInput, + gap: Gap, + syntax: Option, +) -> Result> { + let gap = wire::Gap { + old_start: line_number("old_start", gap.old_start)?, + new_start: line_number("new_start", gap.new_start)?, + len: line_number("len", gap.len)?, + }; + let wire = wire_settings(syntax, None)?; + let input = file_input(file)?; + let (deferred, promise) = env.create_deferred()?; + let work: queue::Work = Box::new(move || { + let rows = wire::expand(&input, &gap, &wire.options(grammars())); + serde_json::to_string(&rows).map_err(json_error) + }); + std::thread::Builder::new() + .name("diffy-node-expand".to_owned()) + .stack_size(queue::STACK_BYTES) + .spawn(move || queue::settle(deferred, work)) + .map_err(|error| Error::from_reason(format!("could not start a thread: {error}")))?; + Ok(promise) +} + +/// [`WireOptions`] without the borrowed store, so it can cross threads. +#[derive(Clone, Copy)] +struct WireSettings { + syntax: bool, + syntax_max_bytes: usize, +} + +impl WireSettings { + fn options<'a>(&self, grammars: &'a GrammarStore) -> WireOptions<'a> { + WireOptions { + syntax: self.syntax.then_some(grammars), + syntax_max_bytes: self.syntax_max_bytes, + ..WireOptions::default() + } + } +} + +fn wire_settings(syntax: Option, max_bytes: Option) -> Result { + let syntax_max_bytes = match max_bytes { + Some(bytes) if bytes.is_finite() && bytes >= 0.0 && bytes.fract() == 0.0 => { + // `as` saturates; usize::MAX means no limit anyway. + bytes as usize + } + Some(bytes) => return Err(invalid(format!("syntax_max_bytes: {bytes} is not a size"))), + None => wire::SYNTAX_MAX_BYTES, + }; + Ok(WireSettings { + syntax: syntax.unwrap_or(false), + syntax_max_bytes, + }) +} + +fn file_input(file: FileInput) -> Result { + let status = match file.status.as_str() { + "modified" => FileStatus::Modified, + "added" => FileStatus::Added, + "deleted" => FileStatus::Deleted, + "renamed" => FileStatus::Renamed, + "copied" => FileStatus::Copied, + other => return Err(invalid(format!("status: unknown file status {other:?}"))), + }; + if file.old_path.is_none() && file.new_path.is_none() { + return Err(invalid("a file needs old_path or new_path".to_owned())); + } + Ok(diffy_compare::FileInput { + status, + old_path: file.old_path, + new_path: file.new_path, + old: bytes(file.old), + new: bytes(file.new), + }) +} + +fn bytes(content: Option>) -> Vec { + match content { + Some(Either::A(array)) => array.to_vec(), + Some(Either::B(text)) => text.into_bytes(), + None => Vec::new(), + } +} + +fn duration(name: &str, ms: f64) -> Result { + Duration::try_from_secs_f64(ms / 1000.0) + .map_err(|_| invalid(format!("{name}: {ms} is not a duration in milliseconds"))) +} + +fn line_number(name: &str, value: f64) -> Result { + if value.is_finite() && value >= 0.0 && value.fract() == 0.0 && value <= f64::from(u32::MAX) { + Ok(value as u32) + } else { + Err(invalid(format!("{name}: {value} is not a line number"))) + } +} + +fn invalid(reason: String) -> Error { + Error::new(Status::InvalidArg, reason) +} + +/// `index.js` turns the `Cancelled` code into an `AbortError`. +fn compare_error(error: diffy_compare::Error) -> Error { + let status = match error { + diffy_compare::Error::Cancelled => Status::Cancelled, + _ => Status::GenericFailure, + }; + Error::new(status, error.to_string()) +} + +fn json_error(error: serde_json::Error) -> Error { + Error::new(Status::GenericFailure, error.to_string()) +} diff --git a/bindings/node/src/queue.rs b/bindings/node/src/queue.rs new file mode 100644 index 00000000..45c5d937 --- /dev/null +++ b/bindings/node/src/queue.rs @@ -0,0 +1,203 @@ +//! Process-wide FIFO of `diff` requests. +//! +//! At most [`limit`](Queue::limit) run at once, each on its own thread. A +//! queued request holds no thread: a finishing thread takes the next one or +//! exits. A cancelled queued request is rejected at once and never starts. + +use std::collections::VecDeque; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; + +use napi::{Env, Error, JsDeferred, Result, Status}; + +/// The semantic diff may run on this thread; same stack as the app's +/// semantic workers. Reserved, not committed. +pub(crate) const STACK_BYTES: usize = 16 << 20; + +/// `diffy_compare`'s per-request worker fan-out. +const WORKERS_PER_REQUEST: usize = 4; + +/// A promise settled with a JSON string from any thread. +pub(crate) type Deferred = JsDeferred Result + Send>>; + +pub(crate) type Work = Box Result + Send>; + +pub(crate) struct Job { + pub deferred: Deferred, + pub cancel: Arc, + pub work: Work, +} + +struct Queue { + /// 0 until first used. + limit: usize, + running: usize, + waiting: VecDeque, +} + +static QUEUE: Mutex = Mutex::new(Queue { + limit: 0, + running: 0, + waiting: VecDeque::new(), +}); + +fn lock() -> MutexGuard<'static, Queue> { + // Nothing panics while the lock is held, so the queue is consistent. + QUEUE.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl Queue { + fn limit(&mut self) -> usize { + if self.limit == 0 { + self.limit = default_limit(); + } + self.limit + } + + /// The oldest waiting job, counted as running, if a slot is free. + /// Cancelled jobs found on the way are rejected. + fn next(&mut self) -> Option { + if self.running >= self.limit() { + return None; + } + while let Some(job) = self.waiting.pop_front() { + if job.cancel.load(Ordering::Relaxed) { + reject_cancelled(job); + continue; + } + self.running += 1; + return Some(job); + } + None + } +} + +/// Fills the cores without requests time-slicing away each other's file +/// budgets. +fn default_limit() -> usize { + let cores = std::thread::available_parallelism().map_or(1, usize::from); + (cores / WORKERS_PER_REQUEST).max(1) +} + +pub(crate) fn submit(job: Job) { + let mut queue = lock(); + queue.waiting.push_back(job); + let ready = queue.next(); + drop(queue); + if let Some(job) = ready { + start(job); + } +} + +/// Raising the limit starts queued jobs; lowering it lets running ones finish. +pub(crate) fn set_limit(limit: usize) { + let mut queue = lock(); + queue.limit = limit.max(1); + let mut ready = Vec::new(); + while let Some(job) = queue.next() { + ready.push(job); + } + drop(queue); + for job in ready { + start(job); + } +} + +pub(crate) struct State { + pub limit: usize, + pub running: usize, + pub queued: usize, +} + +pub(crate) fn state() -> State { + let mut queue = lock(); + State { + limit: queue.limit(), + running: queue.running, + queued: queue.waiting.len(), + } +} + +/// Rejects queued jobs with this flag; running ones poll it themselves. +pub(crate) fn cancel(cancel: &Arc) { + let mut queue = lock(); + let waiting = std::mem::take(&mut queue.waiting); + for job in waiting { + if Arc::ptr_eq(&job.cancel, cancel) { + reject_cancelled(job); + } else { + queue.waiting.push_back(job); + } + } +} + +fn reject_cancelled(job: Job) { + job.deferred + .reject(Error::new(Status::Cancelled, "cancelled".to_owned())); +} + +/// Gives up the caller's slot, or hands it the next queued job. +fn finish() -> Option { + let mut queue = lock(); + queue.running = queue.running.saturating_sub(1); + queue.next() +} + +/// Runs `job`, which already holds a slot, then queued jobs, on a new thread. +fn start(mut job: Job) { + loop { + // Sent only once the thread exists: a failed spawn drops its closure, + // and the promise must still be settled. + let error = match spawn_runner() { + Ok(sender) => match sender.send(job) { + Ok(()) => return, + Err(mpsc::SendError(unsent)) => { + job = unsent; + "it exited at once".to_owned() + } + }, + Err(error) => error.to_string(), + }; + job.deferred.reject(Error::from_reason(format!( + "could not start a thread: {error}" + ))); + match finish() { + Some(next) => job = next, + None => return, + } + } +} + +fn spawn_runner() -> std::io::Result> { + let (sender, receiver) = mpsc::sync_channel::(1); + std::thread::Builder::new() + .name("diffy-node-diff".to_owned()) + .stack_size(STACK_BYTES) + .spawn(move || { + let mut next = receiver.recv().ok(); + while let Some(job) = next { + settle(job.deferred, job.work); + next = finish(); + } + })?; + Ok(sender) +} + +/// A panic in `work` rejects the promise instead of unwinding into Node. +pub(crate) fn settle(deferred: Deferred, work: Work) { + match catch_unwind(AssertUnwindSafe(work)) { + Ok(Ok(json)) => deferred.resolve(Box::new(move |_| Ok(json))), + Ok(Err(error)) => deferred.reject(error), + Err(panic) => deferred.reject(panic_error(panic.as_ref())), + } +} + +fn panic_error(panic: &(dyn std::any::Any + Send)) -> Error { + let message = panic + .downcast_ref::<&str>() + .copied() + .or_else(|| panic.downcast_ref::().map(String::as_str)) + .unwrap_or("unknown panic"); + Error::new(Status::GenericFailure, format!("diffy panicked: {message}")) +} diff --git a/bindings/node/test/diffy.test.mjs b/bindings/node/test/diffy.test.mjs new file mode 100644 index 00000000..fc6b8ae4 --- /dev/null +++ b/bindings/node/test/diffy.test.mjs @@ -0,0 +1,376 @@ +// npm test (after `npm run build` with a bundle holding rust and typescript) + +import assert from 'node:assert/strict' +import { createRequire } from 'node:module' +import { availableParallelism } from 'node:os' +import { afterEach, describe, it } from 'node:test' + +import { FORMAT_VERSION, configure, diff, diffJson, expand, languages } from '../index.js' + +const native = createRequire(import.meta.url)('../binding.js') + +const OLD_RS = `use std::fmt; + +fn render(width: usize) -> String { + let label = "é😀"; + format!("{label} {width}") +} + +fn unchanged() {} +` +const NEW_RS = OLD_RS.replace('fn render(', 'fn render_preview(').replace('{width}', '{width}!') + +const modified = (path, old, next) => ({ status: 'modified', old_path: path, new_path: path, old, new: next }) + +/** Many files with real edits, so a request runs long enough to abort. */ +function bigRequest(count) { + const body = Array.from({ length: 400 }, (_, i) => `fn f${i}(x: u32) -> u32 { x + ${i} }`).join('\n') + const files = Array.from({ length: count }, (_, i) => + modified(`src/f${i}.rs`, body, body.replaceAll('x + 1', 'x * 2').replace('fn f3(', 'fn g3(')), + ) + return { files } +} + +describe('build', () => { + it('reports the format version', () => { + assert.equal(FORMAT_VERSION, 1) + }) + + it('lists the bundled grammars', () => { + const names = languages() + assert.ok(Array.isArray(names)) + for (const name of ['rust', 'typescript']) assert.ok(names.includes(name), `${name} is bundled`) + }) +}) + +describe('expand', () => { + const file = modified('src/view.rs', OLD_RS, OLD_RS) + + it('returns context rows', async () => { + const rows = await expand(file, { old_start: 3, new_start: 3, len: 2 }) + assert.deepEqual(rows, [ + { kind: 'context', old_line: 3, new_line: 3, text: 'fn render(width: usize) -> String {' }, + { kind: 'context', old_line: 4, new_line: 4, text: ' let label = "é😀";' }, + ]) + }) + + it('adds syntax spans in UTF-16 offsets', async () => { + const [fn, label] = await expand(file, { old_start: 3, new_start: 3, len: 2 }, { syntax: true }) + const at = (row, cls) => row.syntax.filter((s) => s.class === cls).map((s) => row.text.slice(s.start, s.end)) + assert.ok(at(fn, 'keyword').includes('fn')) + assert.ok(at(fn, 'function').includes('render')) + assert.ok(at(label, 'string').some((text) => text.includes('é😀'))) + }) + + it('takes bytes and strings alike', async () => { + const bytes = new TextEncoder().encode(OLD_RS) + // A view into a larger buffer: only the view's bytes count. + const padded = new Uint8Array(bytes.length + 8) + padded.set(bytes, 4) + const view = padded.subarray(4, 4 + bytes.length) + const gap = { old_start: 1, new_start: 1, len: 100 } + const want = await expand(file, gap) + assert.deepEqual(await expand({ ...file, old: bytes, new: Buffer.from(OLD_RS) }, gap), want) + assert.deepEqual(await expand({ ...file, old: view, new: view }, gap), want) + assert.equal(want.length, 8) + }) + + it('stops where the sides differ or end', async () => { + const changed = modified('a.rs', 'a\nb\nc\n', 'a\nB\nc\n') + assert.equal((await expand(changed, { old_start: 1, new_start: 1, len: 3 })).length, 1) + assert.equal((await expand(changed, { old_start: 3, new_start: 3, len: 10 })).length, 1) + assert.deepEqual(await expand(changed, { old_start: 1, new_start: 1, len: 0 }), []) + }) +}) + +describe('abort', () => { + it('rejects without starting when already aborted', async () => { + const controller = new AbortController() + const reason = new Error('navigated away') + controller.abort(reason) + await assert.rejects(diff(bigRequest(1), { signal: controller.signal }), (error) => { + assert.equal(error.name, 'AbortError') + assert.equal(error.code, 'ABORT_ERR') + assert.equal(error.cause, reason) + return true + }) + }) + + it('rejects a running request at once', async () => { + const controller = new AbortController() + const running = diff(bigRequest(64), { signal: controller.signal }) + controller.abort() + await assert.rejects(running, { name: 'AbortError' }) + }) + + it('leaves the binding usable', async () => { + const controller = new AbortController() + const running = diff(bigRequest(256), { signal: controller.signal }) + setImmediate(() => controller.abort()) + await assert.rejects(running, { name: 'AbortError' }) + const rows = await expand(modified('a.rs', 'x\n', 'x\n'), { old_start: 1, new_start: 1, len: 1 }) + assert.equal(rows.length, 1) + }) +}) + +describe('invalid input rejects', () => { + const bad = { + 'no request': () => diff(), + 'null request': () => diff(null), + 'files not an array': () => diff({ files: 'a.rs' }), + 'unknown status': () => diff({ files: [{ status: 'changed', new_path: 'a.rs' }] }), + 'missing status': () => diff({ files: [{ new_path: 'a.rs' }] }), + 'no path': () => diff({ files: [{ status: 'modified', old: 'a', new: 'b' }] }), + 'contents a number': () => diff({ files: [{ status: 'added', new_path: 'a.rs', new: 42 }] }), + 'path a number': () => diff({ files: [{ status: 'added', new_path: 7 }] }), + 'negative budget': () => diff({ files: [], budget_ms: -1 }), + 'NaN file budget': () => diff({ files: [], file_budget_ms: NaN }), + 'fractional syntax_max_bytes': () => diff({ files: [], syntax_max_bytes: 1.5 }), + 'syntax not a boolean': () => diff({ files: [], syntax: 'yes' }), + 'signal not a signal': () => diff({ files: [] }, { signal: {} }), + 'negative gap start': () => expand(modified('a.rs', 'a', 'a'), { old_start: -1, new_start: 1, len: 1 }), + 'fractional gap length': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1, len: 1.5 }), + 'gap missing len': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1 }), + 'no gap': () => expand(modified('a.rs', 'a', 'a')), + 'diffJson null request': () => diffJson(null), + } + for (const [name, call] of Object.entries(bad)) { + it(name, async () => { + const result = call() + assert.ok(result instanceof Promise, 'errors are rejections, not throws') + await assert.rejects(result, (error) => { + assert.ok(error instanceof Error) + assert.equal(error.code, 'InvalidArg', error.message) + return true + }) + }) + } +}) + +describe('diff', () => { + it('diffs a modified file semantically', async () => { + const doc = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) + assert.equal(doc.version, FORMAT_VERSION) + assert.equal(doc.files.length, 1) + const [file] = doc.files + assert.equal(file.status, 'modified') + assert.equal(file.language, 'rs') + assert.equal(file.fallback, undefined) + assert.equal(file.additions, 2) + assert.equal(file.deletions, 2) + const rows = file.hunks.flatMap((hunk) => hunk.rows) + const changed = rows.filter((row) => row.kind === 'modified') + assert.equal(changed.length, 2) + const [renamed] = changed + assert.equal(renamed.old.text, 'fn render(width: usize) -> String {') + assert.equal(renamed.new.text, 'fn render_preview(width: usize) -> String {') + for (const span of renamed.new.changes ?? []) { + assert.ok(span.start < span.end && span.end <= renamed.new.text.length) + } + assert.ok(file.renames?.some((r) => r.old === 'render' && r.new === 'render_preview')) + }) + + it('adds syntax spans on request', async () => { + const plain = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) + const highlighted = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)], syntax: true }) + const spans = (doc) => + doc.files[0].hunks.flatMap((h) => h.rows).flatMap((row) => [row.syntax, row.old?.syntax, row.new?.syntax]) + assert.ok(spans(plain).every((s) => s === undefined)) + assert.ok(spans(highlighted).some((s) => s?.length > 0)) + }) + + it('keeps input order across statuses and kinds', async () => { + const files = [ + { status: 'added', new_path: 'notes.txt', new: 'hello' }, + { status: 'deleted', old_path: 'old.ts', old: 'export const a = 1\n' }, + modified('logo.png', new Uint8Array([0x89, 0x50, 0, 1]), new Uint8Array([0x89, 0x50, 0, 2])), + modified('data.unknownext', 'a\nb\n', 'a\nc\n'), + { status: 'renamed', old_path: 'a.ts', new_path: 'b.ts', old: 'let a = 1\n', new: 'let a = 1\n' }, + modified('bad.rs', Buffer.from([0x66, 0x6e, 0xff, 0x0a]), Buffer.from([0x66, 0x6f, 0xff, 0x0a])), + ] + const doc = await diff({ files }) + assert.deepEqual( + doc.files.map((f) => f.new_path ?? f.old_path), + ['notes.txt', 'old.ts', 'logo.png', 'data.unknownext', 'b.ts', 'bad.rs'], + ) + const [added, deleted, binary, unknown, renamed, invalid] = doc.files + assert.equal(added.status, 'added') + assert.equal(added.additions, 1) + assert.equal(added.new_missing_newline, true) + assert.equal(deleted.deletions, 1) + assert.equal(binary.binary, true) + assert.equal(binary.hunks, undefined) + // The extension may name no language or one with no grammar here. + assert.ok(['no_grammar', 'grammar_unavailable'].includes(unknown.fallback?.kind)) + assert.equal(renamed.status, 'renamed') + assert.equal(renamed.hunks, undefined) + assert.deepEqual(renamed.gap_after, { old_start: 1, new_start: 1, len: 1 }) + const texts = invalid.hunks.flatMap((h) => h.rows).flatMap((row) => [row.old?.text, row.new?.text]) + assert.ok(texts.includes('fn�')) + }) + + it('falls back to line diffs past the request budget', async () => { + const { files } = bigRequest(8) + const doc = await diff({ files, budget_ms: 0 }) + assert.equal(doc.files.length, 8) + for (const file of doc.files) { + assert.equal(file.fallback?.kind, 'request_budget') + assert.ok(file.hunks.length > 0) + } + }) + + it('expands the gaps it returns', async () => { + // Unchanged functions on both ends, so context collapses before and after. + const pad = Array.from({ length: 12 }, (_, i) => `fn pad${i}() {}`).join('\n') + const file = modified('src/view.rs', `${pad}\n${OLD_RS}${pad}\n`, `${pad}\n${NEW_RS}${pad}\n`) + const doc = await diff({ files: [file] }) + const gaps = [...doc.files[0].hunks.map((h) => h.gap_before), doc.files[0].gap_after].filter(Boolean) + assert.equal(gaps.length, 2) + for (const gap of gaps) { + const rows = await expand(file, gap) + assert.equal(rows.length, gap.len) + } + }) + + it('runs requests concurrently', async () => { + const docs = await Promise.all(Array.from({ length: 8 }, () => diff(bigRequest(4)))) + for (const doc of docs) assert.equal(doc.files.length, 4) + }) + + it('stops the native request on cancel', async () => { + const token = new native.CancelToken() + token.cancel() + await assert.rejects(native.diff(bigRequest(256), token), { code: 'Cancelled' }) + }) + + it('finishes when the signal never aborts', async () => { + const controller = new AbortController() + const doc = await diff({ files: [modified('a.rs', 'a\n', 'b\n')] }, { signal: controller.signal }) + assert.equal(doc.files.length, 1) + controller.abort() + }) +}) + +describe('diffJson', () => { + it('resolves to the JSON diff resolves to', async () => { + const request = { files: [modified('src/view.rs', OLD_RS, NEW_RS)], syntax: true } + const json = await diffJson(request) + assert.equal(typeof json, 'string') + assert.deepEqual(JSON.parse(json), await diff(request)) + }) + + it('aborts like diff', async () => { + const controller = new AbortController() + const running = diffJson(bigRequest(64), { signal: controller.signal }) + controller.abort() + await assert.rejects(running, { name: 'AbortError' }) + }) +}) + +// The queue is process-wide; each test restores the default limit. +describe('queue', () => { + const { max_concurrent_requests: initial } = configure() + const small = () => diff({ files: [modified('a.rs', 'a\n', 'b\n')] }) + afterEach(() => configure({ max_concurrent_requests: initial })) + + it('defaults to a quarter of the cores, at least one', () => { + assert.ok(Number.isInteger(initial) && initial >= 1) + assert.ok(initial <= Math.max(1, Math.floor(availableParallelism() / 4))) + }) + + it('runs requests in order, at most the limit at once', async () => { + assert.deepEqual(configure({ max_concurrent_requests: 1 }), { max_concurrent_requests: 1 }) + const order = [] + const done = [diff(bigRequest(64)).then(() => order.push('big'))] + for (let i = 0; i < 4; i++) done.push(small().then(() => order.push(i))) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 4 }) + await Promise.all(done) + assert.deepEqual(order, ['big', 0, 1, 2, 3]) + assert.deepEqual(native.queue(), { limit: 1, running: 0, queued: 0 }) + }) + + it('never runs more than the limit', async () => { + configure({ max_concurrent_requests: 2 }) + let settled = false + const all = Promise.all(Array.from({ length: 6 }, () => diff(bigRequest(16)))).finally(() => { + settled = true + }) + let most = 0 + while (!settled) { + const { running, queued } = native.queue() + most = Math.max(most, running) + assert.ok(running + queued <= 6) + await new Promise((resolve) => setImmediate(resolve)) + } + await all + assert.equal(most, 2) + }) + + it('starts queued requests when the limit rises', async () => { + configure({ max_concurrent_requests: 1 }) + const done = Promise.all(Array.from({ length: 3 }, () => diff(bigRequest(16)))) + assert.equal(native.queue().queued, 2) + configure({ max_concurrent_requests: 3 }) + assert.deepEqual(native.queue(), { limit: 3, running: 3, queued: 0 }) + await done + }) + + it('rejects a request aborted while queued at once, without starting it', async () => { + configure({ max_concurrent_requests: 1 }) + const running = diff(bigRequest(64)) + const controller = new AbortController() + const reason = new Error('navigated away') + const queued = diff(bigRequest(64), { signal: controller.signal }) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 1 }) + controller.abort(reason) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 0 }) + let pending = true + running.finally(() => { + pending = false + }) + await assert.rejects(queued, (error) => { + assert.equal(error.name, 'AbortError') + assert.equal(error.cause, reason) + return true + }) + assert.ok(pending, 'rejected before the running request finished') + await running + }) + + it('counts time queued against budget_ms', async () => { + configure({ max_concurrent_requests: 1 }) + const running = diff(bigRequest(64)) + const doc = await diff({ files: [modified('a.rs', 'fn a() {}\n', 'fn b() {}\n')], budget_ms: 1 }) + assert.equal(doc.files[0].fallback?.kind, 'request_budget') + await running + }) + + it('does not queue expand', async () => { + configure({ max_concurrent_requests: 1 }) + let pending = true + const running = diff(bigRequest(64)).finally(() => { + pending = false + }) + const rows = await expand(modified('a.rs', 'x\n', 'x\n'), { old_start: 1, new_start: 1, len: 1 }) + assert.equal(rows.length, 1) + assert.ok(pending, 'expand finished while a diff held the only slot') + await running + }) + + for (const [name, options] of Object.entries({ + zero: { max_concurrent_requests: 0 }, + fractional: { max_concurrent_requests: 1.5 }, + 'not a number': { max_concurrent_requests: '2' }, + 'not an object': 4, + null: null, + })) { + it(`rejects a ${name} limit`, () => { + assert.throws(() => configure(options), (error) => { + assert.ok(error instanceof TypeError) + assert.equal(error.code, 'InvalidArg') + return true + }) + assert.equal(configure().max_concurrent_requests, initial) + }) + } +}) diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml new file mode 100644 index 00000000..6dd1c5dd --- /dev/null +++ b/crates/diffy-compare/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "diffy-compare" +description = "Headless file comparison for Diffy: semantic diff where a grammar exists, line diff otherwise." +version.workspace = true +edition = "2024" +rust-version = "1.92" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +diffy-diff = { path = "../diffy-diff" } +diffy-semantic = { path = "../diffy-semantic" } +diffy-syntax = { path = "../diffy-syntax", features = ["runtime"] } +# gix's own line diff: matches Git, and the app links one copy. +imara-diff = { package = "gix-imara-diff", version = "0.2.1", default-features = false } +tracing = "0.1.44" +serde = { version = "1.0", default-features = false, features = ["derive", "std"], optional = true } + +[features] +default = [] +# Enables `wire`. +serde = ["dep:serde"] + +[dev-dependencies] +serde_json = "1.0" +# For `diffy_syntax::testing` packs. +diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } diff --git a/crates/diffy-compare/FORMAT.md b/crates/diffy-compare/FORMAT.md new file mode 100644 index 00000000..53b3f108 --- /dev/null +++ b/crates/diffy-compare/FORMAT.md @@ -0,0 +1,371 @@ +# Diffy display format, version 1 + +JSON produced by `diffy_compare::wire` (crate feature `serde`). Layout-neutral: +one row list renders both split and unified views. + +## Versioning + +- `version` changes only on a breaking change: removing or renaming a field or + enum value, changing a field's type, meaning or coordinates, or making an + optional field required. +- Adding an optional field keeps the version, so ignore unknown fields. +- Syntax `class` may gain values without a bump; render unknown ones as plain. + +## Conventions + +- Field names and enum values are `snake_case`. +- Optional fields are omitted when empty: absent string or object means none, + absent array means `[]`, absent boolean means `false`. +- Line numbers are 1-based. +- Ranges are half-open: `start` included, `end` excluded. +- **Span offsets are UTF-16 code units within the row's `text`**: + `text.slice(span.start, span.end)` is the span's text. +- `text` is one display line: no line terminator, no trailing `\r`, invalid + UTF-8 bytes as U+FFFD (`�`). The final newline is reported per file + (`old_missing_newline`, `new_missing_newline`). + +## Document + +| Field | Type | | +| --- | --- | --- | +| `version` | number | The format version, `1`. | +| `files` | File[] | In request order. | + +## File + +| Field | Type | | +| --- | --- | --- | +| `status` | `"modified"` \| `"added"` \| `"deleted"` \| `"renamed"` \| `"copied"` | | +| `old_path` | string? | Absent for an added file. | +| `new_path` | string? | Absent for a deleted file. Show `new_path ?? old_path`. | +| `language` | string? | The path's language tag, as Diffy names grammars: the extension (`rs`, `ts`, `py`), or the lowercased name of a file without one (`dockerfile`). Absent when the path names none, or when `fallback.kind` is `no_grammar`. | +| `binary` | boolean? | A binary file has no `hunks` and no `gap_after`. | +| `additions` | number | Added lines. | +| `deletions` | number | Removed lines. | +| `old_missing_newline` | boolean? | The old file's last line has no newline. | +| `new_missing_newline` | boolean? | The new file's last line has no newline. | +| `fallback` | Fallback? | Present when the file got a line diff instead of a semantic one. | +| `hunks` | Hunk[]? | In file order. | +| `gap_after` | Gap? | Unchanged lines after the last hunk; the whole file when there are no hunks (unchanged, or renamed without edits). | +| `outline` | OutlineItem[]? | Items the semantic diff found changed. | +| `moves` | Move[]? | Code the semantic diff found moved. | +| `renames` | Rename[]? | Identifiers the semantic diff found renamed. | + +Render order: for each hunk, `gap_before`, header, rows; then `gap_after`. + +### Fallback + +| Field | Type | | +| --- | --- | --- | +| `kind` | `"no_grammar"` \| `"grammar_unavailable"` \| `"semantic"` \| `"request_budget"` | `no_grammar`: no language for the path. `grammar_unavailable`: the language's grammar failed to load. `semantic`: the semantic diff gave up, see `reason`. `request_budget`: the request's deadline passed before this file was diffed. | +| `reason` | `"too_large"` \| `"parse_errors"` \| `"error_coverage"` \| `"unparsed"` \| `"budget"` \| `"timeout"`? | Present when `kind` is `semantic`, in order: file too large; too many parse errors; parse errors cover too much; parser gave up; work budget ran out; time budget ran out. | +| `message` | string | For people, e.g. `"exceeded the semantic time budget"`. Wording may change; branch on `kind` and `reason`. | + +## Hunk + +| Field | Type | | +| --- | --- | --- | +| `old_start`, `old_len`, `new_start`, `new_len` | number | The `@@ -old_start,old_len +new_start,new_len @@` numbers. An empty side's start is the line before the hunk (`0` at the top of the file). | +| `header` | string? | The innermost item containing the hunk's first non-blank changed line: a changed item's name from the outline (`render_preview`), else a grammar scope label (`fn render_preview`, `impl Parser`, `parse()`). Absent when nothing contains it, e.g. a top-level statement between functions. | +| `gap_before` | Gap? | Unchanged lines between the previous hunk (or the top of the file) and this one. | +| `rows` | Row[] | In display order; each side's line numbers ascend. | + +## Gap + +Unchanged lines left out of the hunks, identical on both sides; the starts +differ when earlier changes shifted lines. + +| Field | Type | | +| --- | --- | --- | +| `old_start` | number | First line on the old side. | +| `new_start` | number | First line on the new side. | +| `len` | number | Lines in the gap, at least 1. | + +To expand, send a Gap: the whole gap or any sub-range (first 20 lines: +`{old_start, new_start, len: 20}`; last 20: `{old_start: old_start + len - 20, +new_start: new_start + len - 20, len: 20}`). The reply is `context` rows, with +syntax spans if syntax was requested. It can be shorter than `len` when the +range runs past the end of the file or no longer matches it. + +## Row + +Tagged by `kind`. + +### `context` + +An unchanged line. + +| Field | Type | | +| --- | --- | --- | +| `kind` | `"context"` | | +| `old_line` | number | | +| `new_line` | number | | +| `text` | string | | +| `syntax` | SyntaxSpan[]? | From the new file. | + +### `removed`, `added`, `modified` + +| Kind | Fields | +| --- | --- | +| `removed` | `old`: Line | +| `added` | `new`: Line | +| `modified` | `old`: Line, `new`: Line — a removed line beside the added line it became. | + +- Split view: `old` left, `new` right; `removed` leaves the right empty, + `added` the left. +- Unified view: a `modified` row is its `old` line removed, then its `new` line + added. To read like `git diff`, show a run of `modified` rows as all their + old lines, then all their new lines. + +### Line + +One side of a changed row. + +| Field | Type | | +| --- | --- | --- | +| `line` | number | Line number on its side. | +| `text` | string | | +| `changes` | ChangeSpan[]? | What changed within the line. Sorted, never overlapping. | +| `syntax` | SyntaxSpan[]? | From this side's file. | + +Without `changes`, a `removed` or `added` line changed as a whole; a `modified` +line has no word-level detail (too different, too long, or differing only in +what `text` hides: a `\r`, an invalid byte, the final newline). + +### ChangeSpan + +| Field | Type | | +| --- | --- | --- | +| `start`, `end` | number | UTF-16 offsets within the line's `text`. | +| `intensity` | `"novel"` \| `"novel_word"` \| `"unchanged_context"` \| `"renamed"` | `novel`: syntax only on this side (an inserted or deleted expression). `novel_word`: a changed word in an otherwise matched line. `unchanged_context`: unchanged syntax inside a changed line; subdue or skip it. `renamed`: an identifier matched as renamed (see `renames`). | + +Line-diff files get only `novel_word` spans, on `modified` rows. + +### SyntaxSpan + +Present only when syntax was requested, the language has a grammar, and the +side is under the size cap. Text outside every span is plain. + +| Field | Type | | +| --- | --- | --- | +| `start`, `end` | number | UTF-16 offsets within the line's `text`. Sorted, never overlapping. | +| `class` | string | One of `keyword`, `string`, `comment`, `number`, `type`, `function`, `operator`, `punctuation`, `variable`, `constant`, `builtin`, `attribute`, `tag`, `property`, `namespace`, `label`, `preprocessor`. Map each to a theme colour; render any other value as plain. | + +Each whole file is parsed, so lines inside multi-line strings and comments +highlight correctly. Changes and syntax are independent layers: draw the change +background, then the syntax colour. + +## OutlineItem + +New-side items first, then old-side ones. + +| Field | Type | | +| --- | --- | --- | +| `side` | `"old"` \| `"new"` | Which file `line` refers to: `removed` items are old-side, the rest new-side. | +| `line` | number | The item's first line. | +| `end_line` | number? | The item's last line, inclusive. | +| `kind` | string | The grammar's node kind, e.g. `function_item`, `class_declaration`. | +| `name` | string | | +| `change` | `"added"` \| `"removed"` \| `"renamed"` \| `"moved"` \| `"edited"` | `renamed`: only identifiers inside it were renamed. | +| `edits` | number? | Changed lines of an `edited` item. | + +## Move + +`old_len` lines from old line `old_start` moved to `new_len` lines at new line +`new_start`. + +| Field | Type | | +| --- | --- | --- | +| `old_start`, `old_len`, `new_start`, `new_len` | number | | +| `from` | string? | Name of the item the code left. | +| `into` | string? | Name of the item the code arrived in. | +| `item` | string? | Name of the moved item itself, when the code is one. | + +## Rename + +| Field | Type | | +| --- | --- | --- | +| `old`, `new` | string | The identifier before and after. | +| `uses` | number | Occurrences renamed. | + +## TypeScript + +```ts +type Document = { version: 1; files: File[] }; + +type File = { + status: "modified" | "added" | "deleted" | "renamed" | "copied"; + old_path?: string; + new_path?: string; + language?: string; + binary?: boolean; + additions: number; + deletions: number; + old_missing_newline?: boolean; + new_missing_newline?: boolean; + fallback?: Fallback; + hunks?: Hunk[]; + gap_after?: Gap; + outline?: OutlineItem[]; + moves?: Move[]; + renames?: Rename[]; +}; + +type Fallback = { + kind: "no_grammar" | "grammar_unavailable" | "semantic" | "request_budget"; + reason?: "too_large" | "parse_errors" | "error_coverage" | "unparsed" | "budget" | "timeout"; + message: string; +}; + +type Hunk = { + old_start: number; + old_len: number; + new_start: number; + new_len: number; + header?: string; + gap_before?: Gap; + rows: Row[]; +}; + +type Gap = { old_start: number; new_start: number; len: number }; + +type Row = + | { kind: "context"; old_line: number; new_line: number; text: string; syntax?: SyntaxSpan[] } + | { kind: "removed"; old: Line } + | { kind: "added"; new: Line } + | { kind: "modified"; old: Line; new: Line }; + +type Line = { line: number; text: string; changes?: ChangeSpan[]; syntax?: SyntaxSpan[] }; + +type ChangeSpan = { + start: number; // UTF-16 + end: number; + intensity: "novel" | "novel_word" | "unchanged_context" | "renamed"; +}; + +type SyntaxSpan = { start: number; end: number; class: string }; // UTF-16 + +type OutlineItem = { + side: "old" | "new"; + line: number; + end_line?: number; + kind: string; + name: string; + change: "added" | "removed" | "renamed" | "moved" | "edited"; + edits?: number; +}; + +type Move = { + old_start: number; + old_len: number; + new_start: number; + new_len: number; + from?: string; + into?: string; + item?: string; +}; + +type Rename = { old: string; new: string; uses: number }; +``` + +## Example + +A semantic Rust diff without syntax: `render` became `render_preview`, and an +argument changed after non-ASCII text. Lines 1–9 and 14–20 are collapsed. `é` +is one UTF-16 unit and `😀` two, so the changed `2` starts at 16 (byte 19). + +```json +{ + "version": 1, + "files": [ + { + "status": "modified", + "old_path": "src/view.rs", + "new_path": "src/view.rs", + "language": "rs", + "additions": 2, + "deletions": 2, + "hunks": [ + { + "old_start": 10, + "old_len": 4, + "new_start": 10, + "new_len": 4, + "header": "render_preview", + "gap_before": { "old_start": 1, "new_start": 1, "len": 9 }, + "rows": [ + { "kind": "context", "old_line": 10, "new_line": 10, "text": "" }, + { + "kind": "modified", + "old": { + "line": 11, + "text": "fn render() {", + "changes": [{ "start": 3, "end": 9, "intensity": "renamed" }] + }, + "new": { + "line": 11, + "text": "fn render_preview() {", + "changes": [{ "start": 3, "end": 17, "intensity": "renamed" }] + } + }, + { + "kind": "modified", + "old": { + "line": 12, + "text": " draw(\"é😀\", 1);", + "changes": [{ "start": 16, "end": 17, "intensity": "novel_word" }] + }, + "new": { + "line": 12, + "text": " draw(\"é😀\", 2);", + "changes": [{ "start": 16, "end": 17, "intensity": "novel_word" }] + } + }, + { "kind": "context", "old_line": 13, "new_line": 13, "text": "}" } + ] + } + ], + "gap_after": { "old_start": 14, "new_start": 14, "len": 7 }, + "outline": [ + { + "side": "new", + "line": 11, + "end_line": 13, + "kind": "function_item", + "name": "render_preview", + "change": "edited", + "edits": 2 + } + ], + "renames": [{ "old": "render", "new": "render_preview", "uses": 1 }] + }, + { + "status": "added", + "new_path": "notes.txt", + "language": "txt", + "additions": 1, + "deletions": 0, + "new_missing_newline": true, + "hunks": [ + { + "old_start": 0, + "old_len": 0, + "new_start": 1, + "new_len": 1, + "rows": [{ "kind": "added", "new": { "line": 1, "text": "hello" } }] + } + ] + } + ] +} +``` + +With syntax requested, old line 12 above also gains: + +```json +"syntax": [ + { "start": 4, "end": 8, "class": "function" }, + { "start": 9, "end": 14, "class": "string" }, + { "start": 16, "end": 17, "class": "constant" } +] +``` diff --git a/crates/diffy-compare/LICENSE-APACHE b/crates/diffy-compare/LICENSE-APACHE new file mode 100644 index 00000000..9267b22b --- /dev/null +++ b/crates/diffy-compare/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Rohit (seatedro) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-compare/LICENSE-MIT b/crates/diffy-compare/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-compare/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-compare/src/document.rs b/crates/diffy-compare/src/document.rs new file mode 100644 index 00000000..b9df603c --- /dev/null +++ b/crates/diffy-compare/src/document.rs @@ -0,0 +1,369 @@ +//! Semantic rows to a [`DiffDocument`]: hunks, aligned rows and spans. + +use std::collections::HashSet; +use std::ops::Range; + +use diffy_diff::{ + AlignedRow, ChangeIntensity, DiffDocument, DocumentBuilder, FileBuilder, FileMeta, + LineGroupSpan, SemanticSpan, Side, SourceCoverage, TextStore, +}; +use diffy_semantic::{Move, OutlineItem, Rename, Row}; + +use crate::line::line_changes; +use crate::{Error, Result}; + +/// What a semantic diff knows beyond its rows. +#[derive(Default)] +pub struct Semantics { + pub outline: Vec, + pub renames: Vec, + pub moves: Vec, + pub groups: Vec, +} + +/// A file shown whole on one `side`: added or deleted. +pub(crate) fn whole_file( + meta: FileMeta, + old: TextStore, + new: TextStore, + side: Side, +) -> Result { + let lines = match side { + Side::Old => old.line_count(), + Side::New => new.line_count(), + }; + let (old, new) = match side { + Side::Old => (old, TextStore::default()), + Side::New => (TextStore::default(), new), + }; + build(meta, old, new, |file| { + if lines > 0 { + match side { + Side::Old => file.hunk(0, 0).change(lines, 0), + Side::New => file.hunk(0, 0).change(0, lines), + }; + } + }) +} + +pub(crate) fn build( + meta: FileMeta, + old: TextStore, + new: TextStore, + shape: impl FnOnce(&mut FileBuilder<'_>), +) -> Result { + let mut builder = DocumentBuilder::new(); + let mut file = builder.file(meta, SourceCoverage::Full, old, new); + shape(&mut file); + file.finish() + .map_err(|error| Error::Build(error.to_string()))?; + Ok(builder.build()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Context, + Change, +} + +#[derive(Clone, Copy)] +struct Segment { + kind: Kind, + old: u32, + new: u32, + shown: bool, +} + +struct HunkRange { + old: Range, + new: Range, + segments: Range, +} + +/// Drops rows' lines past the end of a side and spans that are empty, +/// out of bounds, split a character, or overlap an earlier span. +pub(crate) fn clean_rows(chunks: Vec>, old: &TextStore, new: &TextStore) -> Vec> { + let side = |store: &TextStore, line: Option, spans: Vec| { + let line = line.filter(|&l| l < store.line_count()); + let spans = line + .and_then(|l| store.display_line(l)) + .map(|text| clean_spans(spans, text)) + .unwrap_or_default(); + (line, spans) + }; + chunks + .into_iter() + .map(|chunk| { + chunk + .into_iter() + .map(|row| { + let [old_spans, new_spans] = row.spans; + let (old_line, old_spans) = side(old, row.old, old_spans); + let (new_line, new_spans) = side(new, row.new, new_spans); + Row { + old: old_line, + new: new_line, + spans: [old_spans, new_spans], + } + }) + .collect() + }) + .collect() +} + +/// `spans` clipped to `text`, sorted, without empty, overlapping, or +/// mid-character spans. +pub fn clean_spans(spans: Vec, text: &str) -> Vec { + let mut out: Vec = spans + .into_iter() + .filter_map(|span| { + let start = span.start as usize; + let end = (span.end as usize).min(text.len()); + (start < end && text.is_char_boundary(start) && text.is_char_boundary(end)).then_some( + SemanticSpan { + start: start as u32, + end: end as u32, + intensity: span.intensity, + }, + ) + }) + .collect(); + out.sort_by_key(|span| (span.start, span.end)); + let mut end = 0; + out.retain(|span| { + let keep = span.start >= end; + if keep { + end = span.end; + } + keep + }); + out +} + +/// One file from cleaned (see [`clean_spans`]) semantic rows; differing lines +/// between rows are line-diffed. +pub fn document_from_rows( + meta: FileMeta, + old: TextStore, + new: TextStore, + chunks: Vec>, + structural: bool, + semantics: Semantics, +) -> Result { + let segments = canonical_segments(&chunks, &old, &new); + let hunks = hunk_ranges(&segments); + let mut rows: Vec> = hunks.iter().map(|_| Vec::new()).collect(); + let mut spans: [Vec<(u32, Vec)>; 2] = [Vec::new(), Vec::new()]; + let mut seen = [HashSet::new(), HashSet::new()]; + for row in chunks.iter().flatten() { + let old_hunk = row.old.and_then(|line| find_hunk(&hunks, Side::Old, line)); + let new_hunk = row.new.and_then(|line| find_hunk(&hunks, Side::New, line)); + if let Some(h) = old_hunk.or(new_hunk) { + let range = &hunks[h]; + let aligned = AlignedRow { + old: row.old.filter(|line| range.old.contains(line)), + new: row.new.filter(|line| range.new.contains(line)), + }; + if aligned.old.is_some() || aligned.new.is_some() { + rows[h].push(aligned); + } + } + for (side, line, hunk) in [ + (Side::Old, row.old, old_hunk), + (Side::New, row.new, new_hunk), + ] { + let s = side as usize; + if let (Some(line), Some(_)) = (line, hunk) + && !row.spans[s].is_empty() + && seen[s].insert(line) + { + spans[s].push((line, row.spans[s].clone())); + } + } + } + build(meta, old, new, |file| { + file.structural(structural); + file.outline(semantics.outline); + file.renames(semantics.renames); + file.moves(semantics.moves); + file.groups(semantics.groups); + for (hunk, hunk_rows) in hunks.iter().zip(rows) { + file.hunk(hunk.old.start, hunk.new.start); + for segment in &segments[hunk.segments.clone()] { + match segment.kind { + Kind::Context => file.context(segment.old), + Kind::Change => file.change(segment.old, segment.new), + }; + } + for row in hunk_rows { + file.row(row); + } + } + for side in [Side::Old, Side::New] { + for (line, line_spans) in &spans[side as usize] { + file.spans(side, *line, line_spans); + } + } + }) +} + +fn is_context_row(row: &Row, old: &TextStore, new: &TextStore, lo: u32, ln: u32) -> bool { + old.line(lo) == new.line(ln) + && row + .spans + .iter() + .flatten() + .all(|span| span.intensity == ChangeIntensity::UnchangedContext) +} + +fn canonical_segments(chunks: &[Vec], old: &TextStore, new: &TextStore) -> Vec { + let mut segments = Vec::new(); + let (mut o, mut n) = (0u32, 0u32); + for chunk in chunks { + let mut first = true; + for row in chunk { + let lo = row.old.filter(|&line| line >= o); + let ln = row.new.filter(|&line| line >= n); + if lo.is_none() && ln.is_none() { + continue; + } + let (to, tn) = (lo.unwrap_or(o), ln.unwrap_or(n)); + push_unmentioned(&mut segments, old, new, o..to, n..tn, !first); + first = false; + let kind = match (lo, ln) { + (Some(a), Some(b)) if is_context_row(row, old, new, a, b) => Kind::Context, + _ => Kind::Change, + }; + segments.push(Segment { + kind, + old: u32::from(lo.is_some()), + new: u32::from(ln.is_some()), + shown: true, + }); + o = to + u32::from(lo.is_some()); + n = tn + u32::from(ln.is_some()); + } + } + push_unmentioned( + &mut segments, + old, + new, + o..old.line_count(), + n..new.line_count(), + false, + ); + segments +} + +fn push_unmentioned( + segments: &mut Vec, + old: &TextStore, + new: &TextStore, + old_lines: Range, + new_lines: Range, + inside: bool, +) { + let (old_len, new_len) = (old_lines.len() as u32, new_lines.len() as u32); + if old_len == 0 && new_len == 0 { + return; + } + let equal = old_len == new_len + && old_lines + .clone() + .zip(new_lines.clone()) + .all(|(a, b)| old.line(a) == new.line(b)); + if equal { + segments.push(Segment { + kind: Kind::Context, + old: old_len, + new: new_len, + shown: inside, + }); + return; + } + let changes = line_changes( + lines_bytes(old, old_lines.clone()), + lines_bytes(new, new_lines.clone()), + ); + let mut pieces = Vec::with_capacity(changes.len() * 2 + 1); + let (mut o, mut n) = (0u32, 0u32); + for change in &changes { + if change.old.start > o { + pieces.push((Kind::Context, change.old.start - o, change.new.start - n)); + } + pieces.push(( + Kind::Change, + change.old.len() as u32, + change.new.len() as u32, + )); + o = change.old.end; + n = change.new.end; + } + if o < old_len { + pieces.push((Kind::Context, old_len - o, new_len - n)); + } + let last = pieces.len().saturating_sub(1); + for (index, (kind, old, new)) in pieces.into_iter().enumerate() { + let edge = index == 0 || index == last; + segments.push(Segment { + kind, + old, + new, + shown: kind == Kind::Change || !edge || inside, + }); + } +} + +fn lines_bytes(store: &TextStore, lines: Range) -> &[u8] { + let text = store.as_str(); + let offset = |line: u32| { + store + .line_range(line) + .map_or(text.len(), |range| range.start) + }; + &text.as_bytes()[offset(lines.start)..offset(lines.end)] +} + +fn hunk_ranges(segments: &[Segment]) -> Vec { + let mut hunks = Vec::new(); + let (mut o, mut n) = (0u32, 0u32); + let mut open: Option<(u32, u32, usize, bool)> = None; + for (index, segment) in segments.iter().enumerate() { + let in_hunk = segment.shown || segment.kind == Kind::Change; + match (&mut open, in_hunk) { + (None, true) => open = Some((o, n, index, segment.kind == Kind::Change)), + (Some(state), true) => state.3 |= segment.kind == Kind::Change, + (Some(_), false) => close(&mut hunks, open.take(), o, n, index), + (None, false) => {} + } + o += segment.old; + n += segment.new; + } + close(&mut hunks, open, o, n, segments.len()); + hunks +} + +fn close( + hunks: &mut Vec, + open: Option<(u32, u32, usize, bool)>, + o: u32, + n: u32, + end: usize, +) { + if let Some((start_o, start_n, start, true)) = open { + hunks.push(HunkRange { + old: start_o..o, + new: start_n..n, + segments: start..end, + }); + } +} + +fn find_hunk(hunks: &[HunkRange], side: Side, line: u32) -> Option { + let range = |h: &HunkRange| match side { + Side::Old => h.old.clone(), + Side::New => h.new.clone(), + }; + let index = hunks.partition_point(|h| range(h).end <= line); + (index < hunks.len() && range(&hunks[index]).contains(&line)).then_some(index) +} diff --git a/crates/diffy-compare/src/file.rs b/crates/diffy-compare/src/file.rs new file mode 100644 index 00000000..533d8da8 --- /dev/null +++ b/crates/diffy-compare/src/file.rs @@ -0,0 +1,154 @@ +//! One file's chain: trivial cases, then the semantic diff under its +//! budget, else a line diff. + +use std::time::{Duration, Instant}; + +use diffy_diff::{DiffDocument, FileMeta, FileStatus, LineGroupSpan, Side, TextStore}; +use diffy_semantic::Limits; +use diffy_syntax::LanguageId; + +use crate::document::{Semantics, build, clean_rows, document_from_rows, whole_file}; +use crate::grammar::SemanticGrammar; +use crate::line::{binary_document, line_diff}; +use crate::{Fallback, FileDiff, FileInput, Options, Result, is_binary, semantic_tag}; + +/// A file before any diff runs: either already done (binary, added, +/// deleted, unchanged) or two texts to diff. +pub enum Prepared { + Done(Box), + Text { + meta: FileMeta, + old: TextStore, + new: TextStore, + }, +} + +/// Settles files that need no diff. +pub fn prepare(input: &FileInput) -> Result { + let meta = input.meta(); + if is_binary(&input.old) || is_binary(&input.new) { + return Ok(Prepared::Done(Box::new(binary_document(meta)?))); + } + let old = TextStore::from_bytes(&input.old); + let new = TextStore::from_bytes(&input.new); + let done = if input.status == FileStatus::Added || input.old_path.is_none() { + whole_file(meta, old, new, Side::New)? + } else if input.status == FileStatus::Deleted || input.new_path.is_none() { + whole_file(meta, old, new, Side::Old)? + } else if input.old == input.new { + build(meta, old, new, |_| {})? + } else { + return Ok(Prepared::Text { meta, old, new }); + }; + Ok(Prepared::Done(Box::new(done))) +} + +pub(crate) fn diff_one( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + let (meta, old, new) = match prepare(input)? { + Prepared::Done(document) => { + return Ok(FileDiff { + document: *document, + fallback: None, + }); + } + Prepared::Text { meta, old, new } => (meta, old, new), + }; + let line = |meta: FileMeta, fallback: Fallback| { + Ok(FileDiff { + document: line_diff(meta, &input.old, &input.new)?, + fallback: Some(fallback), + }) + }; + let Some(tag) = semantic_tag(input.display_path()) else { + return line(meta, Fallback::NoGrammar); + }; + if budget(options).is_none() { + return line(meta, Fallback::RequestBudget); + } + let Some(grammar) = grammars(&tag) else { + return line(meta, Fallback::GrammarUnavailable(tag.to_string())); + }; + // A first grammar load can be slow; re-check the budget. + let Some(max_time) = budget(options) else { + return line(meta, Fallback::RequestBudget); + }; + let limits = Limits { + max_time, + ..Limits::default() + }; + let embed = |tree: &diffy_semantic::Tree, source: &str| { + let Some(injector) = &grammar.injector else { + return Vec::new(); + }; + // Regions repeat a few languages many times; look each up once. + let mut seen: Vec<(LanguageId, Option)> = Vec::new(); + let mut out = Vec::new(); + for (language, range) in injector.find(tree, source) { + let at = match seen.iter().position(|(l, _)| *l == language) { + Some(at) => at, + None => { + let found = grammars(&language); + seen.push((language, found)); + seen.len() - 1 + } + }; + if let Some(embedded) = &seen[at].1 { + out.push(diffy_semantic::Embedded { + range, + language: embedded.language.clone(), + rules: embedded.rules.clone(), + name: embedded.name.clone(), + }); + } + } + out + }; + let diff = diffy_semantic::diff_with( + &grammar.language, + &grammar.rules, + old.as_str(), + new.as_str(), + &limits, + options.vet, + Some(&embed), + ); + match diff { + Ok(diff) => { + let chunks = clean_rows(diff.chunks, &old, &new); + let semantics = Semantics { + outline: diff.outline, + renames: diff.renames, + moves: diff.moves, + groups: diff + .groups + .into_iter() + .map(|g| LineGroupSpan { + old: g.old, + new: g.new, + }) + .collect(), + }; + Ok(FileDiff { + document: document_from_rows(meta, old, new, chunks, true, semantics)?, + fallback: None, + }) + } + Err(fallback) => line(meta, Fallback::Semantic(fallback)), + } +} + +/// The semantic time budget for a file starting now, or `None` once the +/// request's deadline has passed. +fn budget(options: &Options<'_>) -> Option { + let Some(deadline) = options.deadline else { + return Some(options.file_budget); + }; + deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .map(|left| left.min(options.file_budget)) +} diff --git a/crates/diffy-compare/src/grammar.rs b/crates/diffy-compare/src/grammar.rs new file mode 100644 index 00000000..be0212bf --- /dev/null +++ b/crates/diffy-compare/src/grammar.rs @@ -0,0 +1,71 @@ +//! Grammars for the semantic diff, from the caller's store. + +use std::sync::{Arc, Mutex, PoisonError}; + +use diffy_semantic::{Queries, Rules}; +use diffy_syntax::{GrammarQueries, GrammarStore, Injector, LanguageId}; + +/// What the semantic diff needs from one language's grammar. +pub struct SemanticGrammar { + pub name: Arc, + pub language: diffy_semantic::Language, + pub rules: Arc, + pub injector: Option, +} + +/// The grammar `store` has for `tag`, with its semantic rules compiled +/// once per loaded grammar. +pub fn semantic_grammar(store: &GrammarStore, tag: &LanguageId) -> Option { + static RULES: Mutex, Arc)>> = Mutex::new(Vec::new()); + let (language, queries) = store.parser(tag.as_str())?; + let cached = |rules: &[(Arc, Arc)]| { + rules + .iter() + .find(|(highlights, _)| Arc::ptr_eq(highlights, &queries.highlights)) + .map(|(_, rules)| rules.clone()) + }; + let found = cached(&RULES.lock().unwrap_or_else(PoisonError::into_inner)); + let rules = match found { + Some(rules) => rules, + None => { + let rules = Arc::new(compile_rules(&language, &queries)); + let mut all = RULES.lock().unwrap_or_else(PoisonError::into_inner); + match cached(&all) { + Some(rules) => rules, + None => { + all.push((queries.highlights.clone(), rules.clone())); + rules + } + } + } + }; + Some(SemanticGrammar { + injector: store.injector(tag.as_str()), + name: queries.language, + language, + rules, + }) +} + +fn compile_rules(language: &diffy_semantic::Language, grammar: &GrammarQueries) -> Rules { + let queries = Queries { + highlights: &grammar.highlights, + locals: grammar.locals.as_deref(), + }; + let own = [ + diffy_semantic::semantic_query(&grammar.language), + grammar.semantic.as_deref(), + ]; + for semantic in own.into_iter().flatten() { + match Rules::new(language, semantic, &queries) { + Ok(rules) => return rules, + Err(error) => tracing::warn!( + target: "diffy::semantic", + language = %grammar.language, + %error, + "semantic query does not compile" + ), + } + } + Rules::derived(language, &queries) +} diff --git a/crates/diffy-compare/src/lib.rs b/crates/diffy-compare/src/lib.rs new file mode 100644 index 00000000..ac814eb7 --- /dev/null +++ b/crates/diffy-compare/src/lib.rs @@ -0,0 +1,216 @@ +//! Headless file comparison: changed files' bytes to [`DiffDocument`]s, with +//! no UI, Git, or network dependencies. +//! +//! [`diff_files`] runs the semantic diff with the path's grammar, falling back +//! to a line diff ([`Fallback`]) when there is none or the semantic diff gives +//! up. The `serde` feature adds [`wire`], a versioned display format. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::time::{Duration, Instant}; + +pub use diffy_diff::{DiffDocument, FileMeta, FileStatus}; +pub use diffy_syntax::{GrammarStore, LanguageId}; + +mod document; +mod file; +mod grammar; +mod line; +mod run; +#[cfg(test)] +mod tests; +#[cfg(feature = "serde")] +pub mod wire; + +#[doc(hidden)] +pub use document::{Semantics, clean_spans, document_from_rows}; +#[doc(hidden)] +pub use file::{Prepared, prepare}; +pub use grammar::{SemanticGrammar, semantic_grammar}; +pub use line::{CONTEXT_LINES, binary_document, line_diff}; + +/// Default [`Options::file_budget`]. +pub const FILE_BUDGET: Duration = Duration::from_millis(400); + +/// One changed file. +#[derive(Debug, Clone, Default)] +pub struct FileInput { + pub status: FileStatus, + /// `None` for an added file. + pub old_path: Option, + /// `None` for a deleted file. + pub new_path: Option, + pub old: Vec, + pub new: Vec, +} + +impl FileInput { + /// The new path, else the old one. + pub fn display_path(&self) -> &str { + self.new_path + .as_deref() + .or(self.old_path.as_deref()) + .unwrap_or_default() + } + + /// Paths and status as document metadata. + pub fn meta(&self) -> FileMeta { + FileMeta { + old_path: self.old_path.as_deref().map(Arc::from), + new_path: self.new_path.as_deref().map(Arc::from), + status: self.status, + ..FileMeta::default() + } + } +} + +/// Why a file got a line diff instead of a semantic one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Fallback { + /// No language is known for the path. + NoGrammar, + /// The store cannot load the grammar for this language tag. + GrammarUnavailable(String), + /// The semantic diff gave up. + Semantic(diffy_semantic::Fallback), + /// The request's deadline passed before this file was diffed. + RequestBudget, +} + +impl std::fmt::Display for Fallback { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoGrammar => f.write_str("no grammar for this file"), + Self::GrammarUnavailable(tag) => write!(f, "{tag} grammar unavailable"), + Self::Semantic(fallback) => fallback.fmt(f), + Self::RequestBudget => f.write_str("request time budget exhausted"), + } + } +} + +/// One file's result. Binary, added, deleted and unchanged files skip the +/// semantic diff and have no fallback. +#[derive(Debug, Clone)] +pub struct FileDiff { + /// Holds exactly one file. + pub document: DiffDocument, + pub fallback: Option, +} + +/// Per-file hooks, called from worker threads. +pub trait Observer: Sync { + /// Runs `work`, the diff of `path`; wrap it to time or attribute it. + fn file(&self, path: &str, work: &mut dyn FnMut()) { + let _ = path; + work(); + } + + /// `seen` of `total` files are done. + fn progress(&self, seen: u32, total: u32) { + let _ = (seen, total); + } +} + +pub struct Options<'a> { + /// Grammars for the semantic diff; [`GrammarStore::none`] gives line + /// diffs only. + pub grammars: &'a GrammarStore, + /// Per-file semantic diff budget. + pub file_budget: Duration, + /// When the whole request must be done. Files started after it get + /// line diffs ([`Fallback::RequestBudget`]); a file's semantic budget + /// never runs past it. + pub deadline: Option, + /// Set to stop the request; checked between files. + pub cancel: Option<&'a AtomicBool>, + /// Confirms identifier renames; see [`diffy_semantic::Vet`]. + pub vet: Option<&'a diffy_semantic::Vet<'a>>, + /// Worker threads; 0 picks from the file count and the machine. + pub workers: usize, + pub observer: Option<&'a dyn Observer>, +} + +impl<'a> Options<'a> { + pub fn new(grammars: &'a GrammarStore) -> Self { + Self { + grammars, + file_budget: FILE_BUDGET, + deadline: None, + cancel: None, + vet: None, + workers: 0, + observer: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + /// [`Options::cancel`] was set. + Cancelled, + /// A document failed to build (a bug). + Build(String), + /// A worker thread could not start or panicked. + Worker(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Cancelled => f.write_str("compare cancelled"), + Self::Build(error) => write!(f, "diff document failed to build: {error}"), + Self::Worker(error) => write!(f, "compare worker failed: {error}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Diffs one file on the calling thread. +pub fn diff_file(input: &FileInput, options: &Options<'_>) -> Result { + diff_file_with(input, options, &|tag| { + semantic_grammar(options.grammars, tag) + }) +} + +/// [`diff_file`] with a custom grammar lookup. +#[doc(hidden)] +pub fn diff_file_with( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + if run::cancelled(options) { + return Err(Error::Cancelled); + } + let diff = run::observed(input, options, grammars)?; + if let Some(observer) = options.observer { + observer.progress(1, 1); + } + Ok(diff) +} + +/// Diffs files in parallel; results are in input order. +pub fn diff_files(inputs: &[FileInput], options: &Options<'_>) -> Result> { + run::diff_all(inputs, options, &|tag| { + semantic_grammar(options.grammars, tag) + }) +} + +/// The language tag the semantic diff uses for `path`: JavaScript is +/// parsed with the TypeScript grammar, JSX with TSX. +pub fn semantic_tag(path: &str) -> Option { + let id = LanguageId::from_path(path)?; + match id.as_str() { + "js" | "mjs" | "cjs" => LanguageId::from_fence("typescript"), + "jsx" => LanguageId::from_fence("tsx"), + _ => Some(id), + } +} + +/// Whether bytes look binary: a NUL in the first 8 KiB, as Git decides. +pub fn is_binary(bytes: &[u8]) -> bool { + bytes.iter().take(8192).any(|byte| *byte == 0) +} diff --git a/crates/diffy-compare/src/line.rs b/crates/diffy-compare/src/line.rs new file mode 100644 index 00000000..eb41c23c --- /dev/null +++ b/crates/diffy-compare/src/line.rs @@ -0,0 +1,54 @@ +//! Line diffs. + +use diffy_diff::{ + DiffDocument, DocumentBuilder, FileMeta, SourceCoverage, TextStore, diff_from_changes, + myers::Change, +}; +use imara_diff::{Algorithm, Diff, InternedInput}; + +use crate::{Error, Result, is_binary}; + +/// Unchanged lines shown around each change in a line diff. +pub const CONTEXT_LINES: u32 = 3; + +/// A line diff of `old` and `new` as one file described by `meta`, or a +/// binary placeholder when either side looks binary. +pub fn line_diff(mut meta: FileMeta, old: &[u8], new: &[u8]) -> Result { + if is_binary(old) || is_binary(new) { + return binary_document(meta); + } + let changes = line_changes(old, new); + let (old, new) = (TextStore::from_bytes(old), TextStore::from_bytes(new)); + meta.binary = false; + diff_from_changes(meta, old, new, &changes, CONTEXT_LINES) + .map_err(|error| Error::Build(error.to_string())) +} + +/// A text-less document for a binary file. +pub fn binary_document(mut meta: FileMeta) -> Result { + meta.binary = true; + let mut builder = DocumentBuilder::new(); + builder + .file( + meta, + SourceCoverage::PatchOnly, + TextStore::default(), + TextStore::default(), + ) + .finish() + .map_err(|error| Error::Build(error.to_string()))?; + Ok(builder.build()) +} + +/// Histogram with Git's slider heuristics, matching `git diff`. +pub(crate) fn line_changes(old: &[u8], new: &[u8]) -> Vec { + let input = InternedInput::new(old, new); + let mut diff = Diff::compute(Algorithm::Histogram, &input); + diff.postprocess_lines(&input); + diff.hunks() + .map(|hunk| Change { + old: hunk.before, + new: hunk.after, + }) + .collect() +} diff --git a/crates/diffy-compare/src/run.rs b/crates/diffy-compare/src/run.rs new file mode 100644 index 00000000..80948c15 --- /dev/null +++ b/crates/diffy-compare/src/run.rs @@ -0,0 +1,148 @@ +//! Running files on worker threads. + +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use diffy_syntax::LanguageId; + +use crate::file::diff_one; +use crate::grammar::SemanticGrammar; +use crate::{Error, FileDiff, FileInput, Options, Result}; + +const MAX_WORKERS: usize = 4; +const MIN_FILES_FOR_PARALLEL: usize = 4; +/// Tree-sitter parsing and the semantic matcher recurse deeply on nested +/// sources, more than a default thread stack allows. +const WORKER_STACK_BYTES: usize = 16 << 20; + +type Grammars<'g> = dyn Fn(&LanguageId) -> Option + Sync + 'g; + +pub(crate) fn cancelled(options: &Options<'_>) -> bool { + options + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) +} + +/// Diffs `input` inside [`crate::Observer::file`]. +pub(crate) fn observed( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + let Some(observer) = options.observer else { + return diff_one(input, options, grammars); + }; + let mut out = None; + observer.file(input.display_path(), &mut || { + out = Some(diff_one(input, options, grammars)); + }); + match out { + Some(out) => out, + // The observer skipped `work`; run it anyway. + None => diff_one(input, options, grammars), + } +} + +/// Diffs `inputs` on workers that take files in order until done or cancelled. +pub(crate) fn diff_all( + inputs: &[FileInput], + options: &Options<'_>, + grammars: &Grammars<'_>, +) -> Result> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + let workers = match options.workers { + 0 => worker_count(inputs.len()), + workers => workers.clamp(1, inputs.len()), + }; + let total = inputs.len() as u32; + let next = AtomicUsize::new(0); + let seen = AtomicU32::new(0); + let mut done = std::thread::scope(|scope| { + let handles = (0..workers) + .map(|index| { + std::thread::Builder::new() + .name(format!("diffy-semantic-{index}")) + .stack_size(WORKER_STACK_BYTES) + .spawn_scoped(scope, || { + let mut done = Vec::new(); + while !cancelled(options) { + let at = next.fetch_add(1, Ordering::Relaxed); + let Some(input) = inputs.get(at) else { + break; + }; + done.push((at, observed(input, options, grammars))); + let seen = seen.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if let Some(observer) = options.observer { + observer.progress(seen, total); + } + } + done + }) + .map_err(|error| Error::Worker(format!("setup failed: {error}"))) + }) + .collect::>>()?; + let mut done = Vec::with_capacity(inputs.len()); + for handle in handles { + let part = handle + .join() + .map_err(|_| Error::Worker("a worker panicked".to_owned()))?; + done.extend(part); + } + Ok::<_, Error>(done) + })?; + if done.len() < inputs.len() { + return Err(Error::Cancelled); + } + done.sort_unstable_by_key(|(at, _)| *at); + done.into_iter().map(|(_, diff)| diff).collect() +} + +/// Runs `work` for `0..len` on workers; results are in index order. The caller +/// works too, so a worker that fails to spawn only costs parallelism; worker +/// panics resume on the caller. +#[cfg(feature = "serde")] +pub(crate) fn map_parallel(len: usize, work: &(dyn Fn(usize) -> T + Sync)) -> Vec { + let next = AtomicUsize::new(0); + let run = || { + let mut done = Vec::new(); + loop { + let at = next.fetch_add(1, Ordering::Relaxed); + if at >= len { + return done; + } + done.push((at, work(at))); + } + }; + let mut done = std::thread::scope(|scope| { + let handles: Vec<_> = (1..worker_count(len)) + .filter_map(|index| { + std::thread::Builder::new() + .name(format!("diffy-wire-{index}")) + .stack_size(WORKER_STACK_BYTES) + .spawn_scoped(scope, run) + .ok() + }) + .collect(); + let mut done = run(); + for handle in handles { + match handle.join() { + Ok(part) => done.extend(part), + Err(panic) => std::panic::resume_unwind(panic), + } + } + done + }); + done.sort_unstable_by_key(|(at, _)| *at); + done.into_iter().map(|(_, value)| value).collect() +} + +fn worker_count(files: usize) -> usize { + if files < MIN_FILES_FOR_PARALLEL { + return 1; + } + let available = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + files.min(available).clamp(1, MAX_WORKERS) +} diff --git a/crates/diffy-compare/src/tests.rs b/crates/diffy-compare/src/tests.rs new file mode 100644 index 00000000..4a42b467 --- /dev/null +++ b/crates/diffy-compare/src/tests.rs @@ -0,0 +1,303 @@ +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::{Duration, Instant}; + +use diffy_diff::{ + BlockKind, ChangeIntensity as Intensity, DiffDocument, FileMeta, FileStatus, SemanticSpan, + Side, TextStore, +}; +use diffy_semantic::Row; + +use crate::document::{Semantics, clean_rows, document_from_rows}; +use crate::{ + Error, Fallback, FileInput, GrammarStore, Observer, Options, diff_file, diff_files, line_diff, +}; + +fn span(start: u32, end: u32, intensity: Intensity) -> SemanticSpan { + SemanticSpan { + start, + end, + intensity, + } +} + +fn line( + old: Option, + new: Option, + old_spans: Vec, + new_spans: Vec, +) -> Row { + Row { + old, + new, + spans: [old_spans, new_spans], + } +} + +fn rows_document(old: &str, new: &str, chunks: Vec>) -> DiffDocument { + let (old, new) = (TextStore::new(old), TextStore::new(new)); + let chunks = clean_rows(chunks, &old, &new); + document_from_rows(meta(), old, new, chunks, true, Semantics::default()).unwrap() +} + +fn meta() -> FileMeta { + FileMeta { + old_path: Some("src/lib.rs".into()), + new_path: Some("src/lib.rs".into()), + status: FileStatus::Modified, + ..FileMeta::default() + } +} + +fn hunks(doc: &DiffDocument) -> Vec<(u32, u32, u32, u32)> { + let h = doc.hunks(); + (0..doc.hunk_count() as usize) + .map(|i| (h.old_start[i], h.old_len[i], h.new_start[i], h.new_len[i])) + .collect() +} + +fn modified(path: &str, old: &str, new: &str) -> FileInput { + FileInput { + status: FileStatus::Modified, + old_path: Some(path.to_owned()), + new_path: Some(path.to_owned()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + } +} + +const OLD_RS: &str = "fn answer() {\n old();\n}\n"; +const NEW_RS: &str = "fn answer() {\n new();\n}\n"; + +#[test] +fn semantic_rows_keep_intensities_pairing_and_full_sources() { + let old = "fn a() {}\nlet x = \"foo\";\nkeep();\n"; + let new = "fn a() {}\nlet x = \"bar\";\nextra();\nkeep();\n"; + let doc = rows_document( + old, + new, + vec![vec![ + line(Some(0), Some(0), vec![], vec![]), + line( + Some(1), + Some(1), + vec![ + span(8, 9, Intensity::UnchangedContext), + span(9, 12, Intensity::NovelWord), + span(12, 13, Intensity::UnchangedContext), + ], + vec![span(9, 12, Intensity::NovelWord)], + ), + line(None, Some(2), vec![], vec![span(0, 7, Intensity::Novel)]), + line(Some(2), Some(3), vec![], vec![]), + ]], + ); + doc.verify_integrity().unwrap(); + assert_eq!(hunks(&doc), [(1, 3, 1, 4)]); + assert_eq!( + doc.blocks().kind, + [BlockKind::Context, BlockKind::Change, BlockKind::Context] + ); + assert_eq!(doc.stats(0).known(), Some((2, 1))); + assert!(doc.prefers_structural(0)); + let intensities: Vec<_> = doc + .semantic_spans(0, Side::Old, 1) + .iter() + .map(|s| s.intensity) + .collect(); + assert_eq!( + intensities, + [ + Intensity::UnchangedContext, + Intensity::NovelWord, + Intensity::UnchangedContext + ] + ); + assert_eq!( + doc.semantic_spans(0, Side::New, 2)[0].intensity, + Intensity::Novel + ); + let rows: Vec<_> = doc.aligned_rows(0).iter().map(|r| (r.old, r.new)).collect(); + assert_eq!( + rows, + [ + (Some(0), Some(0)), + (Some(1), Some(1)), + (None, Some(2)), + (Some(2), Some(3)) + ] + ); + assert_eq!(doc.text(0, Side::New).line_count(), 4); +} + +#[test] +fn lines_that_differ_between_chunks_become_changes() { + let old = "a\nb\nsame\nc\nd\n"; + let new = "A\nb\nsame2\nc\nD\n"; + let doc = rows_document( + old, + new, + vec![ + vec![line( + Some(0), + Some(0), + vec![span(0, 1, Intensity::Novel)], + vec![span(0, 1, Intensity::Novel)], + )], + vec![line( + Some(4), + Some(4), + vec![span(0, 1, Intensity::Novel)], + vec![span(0, 1, Intensity::Novel)], + )], + ], + ); + doc.verify_integrity().unwrap(); + assert_eq!(hunks(&doc), [(1, 1, 1, 1), (3, 1, 3, 1), (5, 1, 5, 1)]); + assert_eq!(doc.stats(0).known(), Some((3, 3))); +} + +#[test] +fn separate_chunks_stay_separate_hunks_and_bad_spans_are_dropped() { + let old = "a\n1\n2\n3\n4\nb\n"; + let new = "x\n1\n2\n3\n4\ny\n"; + let doc = rows_document( + old, + new, + vec![ + vec![line( + Some(0), + Some(0), + vec![span(0, 9, Intensity::Novel)], + vec![], + )], + vec![ + line(Some(5), Some(5), vec![], vec![span(3, 2, Intensity::Novel)]), + line(Some(6), Some(6), vec![], vec![]), + ], + ], + ); + assert_eq!(hunks(&doc), [(1, 1, 1, 1), (6, 1, 6, 1)]); + assert_eq!(doc.semantic_spans(0, Side::Old, 0)[0].end, 1); + assert!(doc.semantic_spans(0, Side::New, 5).is_empty()); +} + +#[test] +fn added_files_skip_the_semantic_diff() { + let store = GrammarStore::none(); + let diff = diff_file( + &FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some("src/new.rs".to_owned()), + old: Vec::new(), + new: b"fn new() {}\nlet x = 1;\n".to_vec(), + }, + &Options::new(&store), + ) + .unwrap(); + let doc = diff.document; + assert_eq!(diff.fallback, None); + assert_eq!(hunks(&doc), [(0, 0, 1, 2)]); + assert_eq!(doc.text(0, Side::New).line(1), Some("let x = 1;")); + assert_eq!(doc.summary(0).status, FileStatus::Added); +} + +#[test] +fn files_without_a_grammar_get_line_diffs() { + let store = GrammarStore::none(); + let input = modified("src/lib.rs", "a\nb\n", "a\nc\n"); + let diff = diff_file(&input, &Options::new(&store)).unwrap(); + let fallback = diff.fallback.unwrap(); + assert!(matches!(fallback, Fallback::GrammarUnavailable(_))); + assert!(fallback.to_string().ends_with(" grammar unavailable")); + assert_eq!(hunks(&diff.document), [(1, 2, 1, 2)]); +} + +#[test] +fn rust_files_get_a_semantic_diff_within_the_budget() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let mut options = Options::new(&store); + options.deadline = Some(Instant::now() + Duration::from_secs(60)); + let diff = diff_file(&modified("src/lib.rs", OLD_RS, NEW_RS), &options).unwrap(); + assert_eq!(diff.fallback, None); + assert!(diff.document.prefers_structural(0)); + assert!(!diff.document.semantic_spans(0, Side::New, 1).is_empty()); +} + +#[test] +fn files_after_the_deadline_get_line_diffs() { + // No grammar is needed: the deadline is checked before one loads. + let store = GrammarStore::none(); + let mut options = Options::new(&store); + options.deadline = Some(Instant::now()); + let inputs = [ + modified("src/lib.rs", OLD_RS, NEW_RS), + FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some("src/new.rs".to_owned()), + new: b"fn new() {}\n".to_vec(), + ..FileInput::default() + }, + modified("src/main.rs", OLD_RS, NEW_RS), + ]; + let diffs = diff_files(&inputs, &options).unwrap(); + let fallbacks: Vec<_> = diffs.iter().map(|diff| diff.fallback.clone()).collect(); + assert_eq!( + fallbacks, + [ + Some(Fallback::RequestBudget), + None, + Some(Fallback::RequestBudget) + ] + ); + let line = line_diff(inputs[0].meta(), OLD_RS.as_bytes(), NEW_RS.as_bytes()).unwrap(); + assert_eq!(hunks(&diffs[0].document), hunks(&line)); + assert!(!diffs[0].document.prefers_structural(0)); +} + +#[test] +fn cancelled_requests_stop() { + let store = GrammarStore::none(); + let cancel = AtomicBool::new(true); + let mut options = Options::new(&store); + options.cancel = Some(&cancel); + let input = modified("src/lib.rs", OLD_RS, NEW_RS); + assert_eq!(diff_file(&input, &options).unwrap_err(), Error::Cancelled); + assert_eq!( + diff_files(&[input.clone(), input], &options).unwrap_err(), + Error::Cancelled + ); +} + +struct CancelAfterFirst<'a> { + cancel: &'a AtomicBool, + files: AtomicU32, +} + +impl Observer for CancelAfterFirst<'_> { + fn file(&self, _path: &str, work: &mut dyn FnMut()) { + self.files.fetch_add(1, Ordering::Relaxed); + work(); + self.cancel.store(true, Ordering::Relaxed); + } +} + +#[test] +fn cancelling_mid_request_stops_picking_up_files() { + let store = GrammarStore::none(); + let cancel = AtomicBool::new(false); + let observer = CancelAfterFirst { + cancel: &cancel, + files: AtomicU32::new(0), + }; + let mut options = Options::new(&store); + options.cancel = Some(&cancel); + options.workers = 1; + options.observer = Some(&observer); + let inputs = vec![modified("a.txt", "a\n", "b\n"); 8]; + assert_eq!(diff_files(&inputs, &options).unwrap_err(), Error::Cancelled); + assert_eq!(observer.files.load(Ordering::Relaxed), 1); +} diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs new file mode 100644 index 00000000..02d147ce --- /dev/null +++ b/crates/diffy-compare/src/wire.rs @@ -0,0 +1,1017 @@ +//! The versioned display format non-Rust clients render diffs from. +//! +//! [`file()`] and [`document`] project [`FileDiff`]s; [`expand`] returns the +//! rows of a collapsed gap. The crate's `FORMAT.md` specifies every field. +//! Lines are one-based; span offsets are half-open UTF-16 code units into +//! [`TextStore::display_line`] text. + +use std::borrow::Cow; +use std::cell::OnceCell; +use std::ops::Range; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use diffy_diff::{ + AlignedRow, BlockKind, ChangeIntensity, DiffDocument, FileStatus, ItemChange, Side as DocSide, + TextStore, inline_diff, +}; +use diffy_syntax::{GrammarStore, HighlightKind, HighlightSpan, LanguageId, ScopeTable}; +use serde::{Deserialize, Serialize}; + +use crate::{FileDiff, FileInput}; + +/// Bump on any breaking change, as `FORMAT.md` defines it. +pub const FORMAT_VERSION: u32 = 1; + +/// Default [`WireOptions::syntax_max_bytes`]. +pub const SYNTAX_MAX_BYTES: usize = 1 << 20; + +/// What a projection adds beyond the diff itself. +#[derive(Debug, Clone, Copy)] +pub struct WireOptions<'a> { + /// Grammars for syntax spans and scope headers; `None` emits neither. + pub syntax: Option<&'a GrammarStore>, + /// Sides longer than this, in bytes, get no syntax spans or scope + /// headers. + pub syntax_max_bytes: usize, + /// Files projected after this get no syntax spans or scope headers. + pub deadline: Option, + /// Once set, the remaining files get no syntax spans or scope headers. + pub cancel: Option<&'a AtomicBool>, +} + +impl Default for WireOptions<'_> { + fn default() -> Self { + Self { + syntax: None, + syntax_max_bytes: SYNTAX_MAX_BYTES, + deadline: None, + cancel: None, + } + } +} + +// ---- Format ------------------------------------------------------------ + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Document { + /// [`FORMAT_VERSION`] of the producer. + pub version: u32, + pub files: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct File { + pub status: Status, + /// `None` for an added file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub old_path: Option, + /// `None` for a deleted file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_path: Option, + /// The path's language tag (`rs`, `dockerfile`); `None` with + /// [`FallbackKind::NoGrammar`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub language: Option, + /// A binary file has no hunks or gaps. + #[serde(default, skip_serializing_if = "is_false")] + pub binary: bool, + pub additions: u32, + pub deletions: u32, + /// The old side's last line has no newline. + #[serde(default, skip_serializing_if = "is_false")] + pub old_missing_newline: bool, + /// The new side's last line has no newline. + #[serde(default, skip_serializing_if = "is_false")] + pub new_missing_newline: bool, + /// Why the file got a line diff instead of a semantic one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hunks: Vec, + /// Unchanged lines after the last hunk; for a file without hunks, the + /// whole file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_after: Option, + /// New-side items, then old-side ones. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outline: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub moves: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub renames: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Status { + Modified, + Added, + Deleted, + Renamed, + Copied, +} + +impl From for Status { + fn from(status: FileStatus) -> Self { + match status { + FileStatus::Modified => Self::Modified, + FileStatus::Added => Self::Added, + FileStatus::Deleted => Self::Deleted, + FileStatus::Renamed => Self::Renamed, + FileStatus::Copied => Self::Copied, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Fallback { + pub kind: FallbackKind, + /// Set when `kind` is `semantic`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// For people; wording may change between releases. + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FallbackKind { + NoGrammar, + GrammarUnavailable, + Semantic, + RequestBudget, +} + +/// Why the semantic diff gave up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SemanticReason { + TooLarge, + ParseErrors, + ErrorCoverage, + Unparsed, + Budget, + Timeout, +} + +impl From<&crate::Fallback> for Fallback { + fn from(fallback: &crate::Fallback) -> Self { + use diffy_semantic::Fallback as Semantic; + let (kind, reason) = match fallback { + crate::Fallback::NoGrammar => (FallbackKind::NoGrammar, None), + crate::Fallback::GrammarUnavailable(_) => (FallbackKind::GrammarUnavailable, None), + crate::Fallback::RequestBudget => (FallbackKind::RequestBudget, None), + crate::Fallback::Semantic(semantic) => { + let reason = match semantic { + Semantic::TooLarge(_) => SemanticReason::TooLarge, + Semantic::ParseErrors(_) => SemanticReason::ParseErrors, + Semantic::ErrorCoverage(_) => SemanticReason::ErrorCoverage, + Semantic::Unparsed => SemanticReason::Unparsed, + Semantic::Budget => SemanticReason::Budget, + Semantic::Timeout => SemanticReason::Timeout, + }; + (FallbackKind::Semantic, Some(reason)) + } + }; + Self { + kind, + reason, + message: fallback.to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Hunk { + /// The `@@ -old_start,old_len +new_start,new_len @@` numbers; an empty + /// side's start is the line before the hunk. + pub old_start: u32, + pub old_len: u32, + pub new_start: u32, + pub new_len: u32, + /// The innermost item around the first change, e.g. `impl Parser`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub header: Option, + /// Unchanged lines between the previous hunk (or the file start) and + /// this one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_before: Option, + pub rows: Vec, +} + +/// Collapsed unchanged lines, the same on both sides. Also the request +/// for [`expand`]: any sub-range of a gap is a valid `Gap`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Gap { + /// One-based first line on each side. + pub old_start: u32, + pub new_start: u32, + pub len: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Row { + /// An unchanged line. `syntax` is the new side's. + Context { + old_line: u32, + new_line: u32, + text: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + syntax: Vec, + }, + Removed { + old: Line, + }, + Added { + new: Line, + }, + /// A removed line beside the added line it became. + Modified { + old: Line, + new: Line, + }, +} + +/// One side's line of a changed row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Line { + /// One-based line number on its side. + pub line: u32, + pub text: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub syntax: Vec, +} + +/// A changed run of a line. Offsets are UTF-16 code units within the +/// line's text; spans are sorted and disjoint. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangeSpan { + pub start: u32, + pub end: u32, + pub intensity: Intensity, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Intensity { + /// Syntax that exists only on this side. + Novel, + /// A changed word inside an otherwise matched line. + NovelWord, + /// Unchanged syntax inside a changed line. + UnchangedContext, + /// An identifier the diff matched as renamed (see [`File::renames`]). + Renamed, +} + +impl From for Intensity { + fn from(intensity: ChangeIntensity) -> Self { + match intensity { + ChangeIntensity::Novel => Self::Novel, + ChangeIntensity::NovelWord => Self::NovelWord, + ChangeIntensity::UnchangedContext => Self::UnchangedContext, + ChangeIntensity::Renamed => Self::Renamed, + } + } +} + +/// A highlighted run of a line, with [`ChangeSpan`]'s offsets and ordering. +/// Text outside spans is plain. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SyntaxSpan { + pub start: u32, + pub end: u32, + /// A [`HighlightKind::name`] such as `keyword` or `string`; clients + /// map it to their theme and treat unknown classes as plain. + pub class: Cow<'static, str>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Side { + Old, + New, +} + +impl From for Side { + fn from(side: DocSide) -> Self { + match side { + DocSide::Old => Self::Old, + DocSide::New => Self::New, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutlineItem { + pub side: Side, + /// One-based first line of the item on `side`. + pub line: u32, + /// One-based last line of the item on `side`, inclusive. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_line: Option, + /// The grammar's item kind, such as `function_item`. + pub kind: String, + pub name: String, + pub change: ItemChangeKind, + /// Changed lines of an `edited` item. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edits: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ItemChangeKind { + Added, + Removed, + Renamed, + Moved, + Edited, +} + +/// Code that moved: `old_len` lines from `old_start` became `new_len` +/// lines at `new_start` (one-based). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Move { + pub old_start: u32, + pub old_len: u32, + pub new_start: u32, + pub new_len: u32, + /// The item the code left. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from: Option, + /// The item the code arrived in. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub into: Option, + /// The moved item itself, when the code is one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub item: Option, +} + +/// An identifier renamed throughout the file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Rename { + pub old: String, + pub new: String, + pub uses: u32, +} + +fn is_false(value: &bool) -> bool { + !*value +} + +// ---- Projection -------------------------------------------------------- + +/// Projects every file; `inputs` and `diffs` pair up by index, and extra +/// entries of the longer slice are ignored. +pub fn document(inputs: &[FileInput], diffs: &[FileDiff], options: &WireOptions<'_>) -> Document { + let len = inputs.len().min(diffs.len()); + // Highlighting parses every file, so it runs on workers. + let files = if options.syntax.is_some() { + crate::run::map_parallel(len, &|at| file(&inputs[at], &diffs[at], options)) + } else { + (0..len) + .map(|at| file(&inputs[at], &diffs[at], options)) + .collect() + }; + Document { + version: FORMAT_VERSION, + files, + } +} + +/// Projects one file. +pub fn file(input: &FileInput, diff: &FileDiff, options: &WireOptions<'_>) -> File { + let late = options + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + || options + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)); + let options = &WireOptions { + syntax: options.syntax.filter(|_| !late), + ..*options + }; + let doc = &diff.document; + let has_file = doc.file_count() > 0; + let binary = crate::is_binary(&input.old) + || crate::is_binary(&input.new) + || doc.files().meta.first().is_some_and(|meta| meta.binary); + let (additions, deletions) = if has_file { + doc.stats(0).known().unwrap_or((0, 0)) + } else { + (0, 0) + }; + let language = match diff.fallback { + Some(crate::Fallback::NoGrammar) => None, + _ => LanguageId::from_path(input.display_path()).map(|id| id.as_str().to_owned()), + }; + let mut out = File { + status: input.status.into(), + old_path: input.old_path.clone(), + new_path: input.new_path.clone(), + language, + binary, + additions, + deletions, + old_missing_newline: false, + new_missing_newline: false, + fallback: diff.fallback.as_ref().map(Fallback::from), + hunks: Vec::new(), + gap_after: None, + outline: Vec::new(), + moves: Vec::new(), + renames: Vec::new(), + }; + if binary || !has_file { + return out; + } + let facts = doc.facts(0); + out.old_missing_newline = facts.old_missing_newline; + out.new_missing_newline = facts.new_missing_newline; + if let Some(semantics) = doc.semantics(0) { + out.outline = semantics.outline().iter().map(outline_item).collect(); + out.moves = semantics.moves().iter().map(wire_move).collect(); + out.renames = semantics + .renames() + .iter() + .map(|rename| Rename { + old: rename.old.to_string(), + new: rename.new.to_string(), + uses: rename.uses, + }) + .collect(); + } + Projector::new(doc, input, options).hunks(&mut out); + out +} + +/// Context rows for `gap`, with new-side syntax when `options` asks for it. +/// Highlighting parses the whole new side so lexical state is right. Stops +/// early where the range leaves either side or the sides differ (stale). +pub fn expand(input: &FileInput, gap: &Gap, options: &WireOptions<'_>) -> Vec { + if gap.len == 0 + || gap.old_start == 0 + || gap.new_start == 0 + || crate::is_binary(&input.old) + || crate::is_binary(&input.new) + { + return Vec::new(); + } + let old = TextStore::from_bytes(&input.old); + let new = TextStore::from_bytes(&input.new); + let (old_first, new_first) = (gap.old_start - 1, gap.new_start - 1); + let len = gap + .len + .min(old.line_count().saturating_sub(old_first)) + .min(new.line_count().saturating_sub(new_first)); + let new_path = input.new_path.as_deref().or(input.old_path.as_deref()); + let syntax = highlights(options, new_path, &new); + let mut rows = Vec::with_capacity(len as usize); + for k in 0..len { + let (o, n) = (old_first + k, new_first + k); + if old.line(o) != new.line(n) { + break; + } + rows.push(context_row(&new, &syntax, n, o + 1, n + 1)); + } + rows +} + +fn outline_item(item: &diffy_diff::OutlineItem) -> OutlineItem { + let (change, edits) = match item.change { + ItemChange::Added => (ItemChangeKind::Added, None), + ItemChange::Removed => (ItemChangeKind::Removed, None), + ItemChange::Renamed => (ItemChangeKind::Renamed, None), + ItemChange::Moved => (ItemChangeKind::Moved, None), + ItemChange::Edited(lines) => (ItemChangeKind::Edited, Some(lines)), + }; + OutlineItem { + side: item.side.into(), + line: item.lines.start + 1, + end_line: Some(item.lines.end), + kind: item.kind.to_string(), + name: item.name.to_string(), + change, + edits, + } +} + +fn wire_move(moved: &diffy_diff::Move) -> Move { + let name = |name: &Option>| name.as_deref().map(str::to_owned); + Move { + old_start: moved.old_line + 1, + old_len: moved.old_lines, + new_start: moved.new_line + 1, + new_len: moved.lines, + from: name(&moved.from), + into: name(&moved.into), + item: name(&moved.item), + } +} + +/// Builds the hunks of file 0 of a one-file document. +struct Projector<'a> { + doc: &'a DiffDocument, + input: &'a FileInput, + options: &'a WireOptions<'a>, + /// The stores hold only the lines the patch shows: no gaps, and no + /// syntax (fragments would parse out of context). + partial: bool, + texts: [&'a TextStore; 2], + syntax: [Vec; 2], + scopes: [OnceCell; 2], +} + +impl<'a> Projector<'a> { + fn new(doc: &'a DiffDocument, input: &'a FileInput, options: &'a WireOptions<'a>) -> Self { + let partial = doc.files().partial[0]; + let texts = [doc.text(0, DocSide::Old), doc.text(0, DocSide::New)]; + let syntax = if partial { + [Vec::new(), Vec::new()] + } else { + [DocSide::Old, DocSide::New] + .map(|side| highlights(options, side_path(input, side), texts[side as usize])) + }; + Self { + doc, + input, + options, + partial, + texts, + syntax, + scopes: [OnceCell::new(), OnceCell::new()], + } + } + + fn hunks(&self, out: &mut File) { + let (doc, partial) = (self.doc, self.partial); + let (h, b) = (doc.hunks(), doc.blocks()); + let structural = doc.prefers_structural(0); + let mut steps = Vec::new(); + // Zero-based ends of the previous hunk on each side. + let (mut old_end, mut new_end) = (0u32, 0u32); + for hunk in doc.files().hunks[0].clone() { + let hi = hunk as usize; + let old_lines = doc.hunk_lines(hunk, DocSide::Old); + let new_lines = doc.hunk_lines(hunk, DocSide::New); + let gap_before = if partial { + None + } else { + gap( + (old_end, new_end), + old_lines.start.saturating_sub(old_end), + new_lines.start.saturating_sub(new_end), + ) + }; + let rows_hint = h.old_len[hi].max(h.new_len[hi]) as usize; + let mut rows = Vec::with_capacity(rows_hint); + let aligned = if structural { + doc.aligned_rows(hunk) + } else { + &[] + }; + for block in h.blocks[hi].clone() { + let bi = block as usize; + let at = BlockAt { + old_store: b.old_store[bi], + new_store: b.new_store[bi], + old_line: b.old_line[bi] - 1, + new_line: b.new_line[bi] - 1, + }; + match b.kind[bi] { + BlockKind::Context => { + for k in 0..b.old_len[bi] { + rows.push(context_row( + self.texts[1], + &self.syntax[1], + at.new_store + k, + at.old_line + k + 1, + at.new_line + k + 1, + )); + } + } + BlockKind::Change => { + let lens = (b.old_len[bi], b.new_len[bi]); + steps.clear(); + if aligned.is_empty() || !structural_steps(aligned, &at, lens, &mut steps) { + steps.clear(); + positional_steps(lens, &mut steps); + } + for &step in &steps { + rows.push(self.change_row(&at, step)); + } + } + } + } + out.hunks.push(Hunk { + old_start: h.old_start[hi], + old_len: h.old_len[hi], + new_start: h.new_start[hi], + new_len: h.new_len[hi], + header: self.header(hunk), + gap_before, + rows, + }); + old_end = old_lines.end; + new_end = new_lines.end; + } + if !partial { + out.gap_after = gap( + (old_end, new_end), + self.texts[0].line_count().saturating_sub(old_end), + self.texts[1].line_count().saturating_sub(new_end), + ); + } + } + + /// The row of one step of a change block; offsets are within the block. + fn change_row(&self, at: &BlockAt, step: Step) -> Row { + let line = |side: DocSide, offset: u32| match side { + DocSide::Old => (at.old_store + offset, at.old_line + offset), + DocSide::New => (at.new_store + offset, at.new_line + offset), + }; + match step { + Step::Paired(o, n) => { + let (old, new) = (line(DocSide::Old, o), line(DocSide::New, n)); + let old_spans = self.doc.semantic_spans(0, DocSide::Old, old.1); + let new_spans = self.doc.semantic_spans(0, DocSide::New, new.1); + if old_spans.is_empty() && new_spans.is_empty() { + // Unmarked by the semantic diff: use the word diff. + let old_text = self.display(DocSide::Old, old.0); + let new_text = self.display(DocSide::New, new.0); + let words = inline_diff(old_text, new_text); + Row::Modified { + old: self.line(DocSide::Old, old, word_spans(&words.old)), + new: self.line(DocSide::New, new, word_spans(&words.new)), + } + } else { + Row::Modified { + old: self.line(DocSide::Old, old, semantic(old_spans)), + new: self.line(DocSide::New, new, semantic(new_spans)), + } + } + } + Step::Removed(o) => { + let old = line(DocSide::Old, o); + let spans = semantic(self.doc.semantic_spans(0, DocSide::Old, old.1)); + Row::Removed { + old: self.line(DocSide::Old, old, spans), + } + } + Step::Added(n) => { + let new = line(DocSide::New, n); + let spans = semantic(self.doc.semantic_spans(0, DocSide::New, new.1)); + Row::Added { + new: self.line(DocSide::New, new, spans), + } + } + } + } + + fn display(&self, side: DocSide, store: u32) -> &str { + self.texts[side as usize].display_line(store).unwrap_or("") + } + + /// One side's line: `(store index, zero-based line)`, and its changed + /// byte ranges within the display text. + fn line( + &self, + side: DocSide, + (store, line): (u32, u32), + changes: impl IntoIterator, Intensity)>, + ) -> Line { + let texts = self.texts[side as usize]; + let text = texts.display_line(store).unwrap_or(""); + let mut units = Utf16::new(text); + let changes = changes + .into_iter() + .filter_map(|(range, intensity)| { + let (start, end) = units.range(range)?; + Some(ChangeSpan { + start, + end, + intensity, + }) + }) + .collect(); + let syntax = syntax_spans(texts, &self.syntax[side as usize], store, &mut units); + Line { + line: line + 1, + text: text.to_owned(), + changes, + syntax, + } + } + + /// The innermost item whose lines contain the hunk's first changed + /// line with text: a changed item of the semantic outline, else a + /// scope of the grammar's scope table. `None` when nothing contains it. + fn header(&self, hunk: u32) -> Option { + let (h, b) = (self.doc.hunks(), self.doc.blocks()); + let hi = hunk as usize; + let first_change = h.blocks[hi] + .clone() + .map(|block| block as usize) + .find(|&bi| b.kind[bi] == BlockKind::Change); + // Zero-based first line of the change on each side, and whether + // the side has changed lines there; a side without them holds the + // change between the anchor line and the one before. + let (old, new) = match first_change { + Some(bi) => { + // Leading blank lines belong to the enclosing scope; skip them. + let first_text = |side: DocSide, store: u32, len: u32| { + (0..len) + .find(|&k| { + self.display(side, store + k) + .bytes() + .any(|byte| !byte.is_ascii_whitespace()) + }) + .unwrap_or(0) + }; + let old_skip = first_text(DocSide::Old, b.old_store[bi], b.old_len[bi]); + let new_skip = first_text(DocSide::New, b.new_store[bi], b.new_len[bi]); + ( + (b.old_line[bi] - 1 + old_skip, b.old_len[bi] > 0), + (b.new_line[bi] - 1 + new_skip, b.new_len[bi] > 0), + ) + } + None => ( + (h.old_start[hi].saturating_sub(1), false), + (h.new_start[hi].saturating_sub(1), true), + ), + }; + // Prefer the new side when it has changed lines. + let (side, other) = if new.1 { + (DocSide::New, DocSide::Old) + } else { + (DocSide::Old, DocSide::New) + }; + let anchor = |side: DocSide| match side { + DocSide::Old => old, + DocSide::New => new, + }; + if let Some(semantics) = self.doc.semantics(0) { + // Edited items are new-side and removed ones old-side, so a side + // with no containing item defers to the other. Items on a side + // nest, so the shortest is innermost. + let innermost = |side: DocSide| { + let (line, lines) = anchor(side); + semantics + .outline() + .iter() + .filter(|item| item.side == side) + .filter(|item| { + if lines { + item.lines.contains(&line) + } else { + item.lines.start < line && line < item.lines.end + } + }) + .min_by_key(|item| item.lines.len()) + }; + if let Some(item) = innermost(side).or_else(|| innermost(other)) { + return Some(item.name.to_string()); + } + } + let (line, _) = anchor(side); + // Sorted by start, longer first, so the last span containing the + // line is the innermost. + self.scopes(side) + .spans() + .iter() + .rfind(|span| span.lines.contains(&line)) + .map(|span| span.label.to_string()) + } + + fn scopes(&self, side: DocSide) -> &ScopeTable { + self.scopes[side as usize].get_or_init(|| { + let text = self.texts[side as usize].as_str(); + let language = side_path(self.input, side).and_then(LanguageId::from_path); + match (self.options.syntax, language) { + (Some(grammars), Some(language)) + if !self.partial && text.len() <= self.options.syntax_max_bytes => + { + diffy_syntax::scope_table(grammars, &language, text) + } + _ => ScopeTable::default(), + } + }) + } +} + +/// Where a block's first line sits: store indices and zero-based lines. +struct BlockAt { + old_store: u32, + new_store: u32, + old_line: u32, + new_line: u32, +} + +/// The gap after zero-based ends `(old, new)`; the sides agree on its +/// length in a valid document, the shorter wins otherwise. +fn gap((old, new): (u32, u32), old_len: u32, new_len: u32) -> Option { + let len = old_len.min(new_len); + (len > 0).then_some(Gap { + old_start: old + 1, + new_start: new + 1, + len, + }) +} + +fn side_path(input: &FileInput, side: DocSide) -> Option<&str> { + let (own, other) = match side { + DocSide::Old => (&input.old_path, &input.new_path), + DocSide::New => (&input.new_path, &input.old_path), + }; + own.as_deref().or(other.as_deref()) +} + +/// A row of a change block; offsets are within the block. +#[derive(Debug, Clone, Copy)] +enum Step { + Paired(u32, u32), + Removed(u32), + Added(u32), +} + +/// Pairs a change block's lines as the semantic diff aligned them; `false` +/// unless the rows cover the block in order. +fn structural_steps( + rows: &[AlignedRow], + at: &BlockAt, + (old_len, new_len): (u32, u32), + steps: &mut Vec, +) -> bool { + let offset = |line: Option, first: u32, len: u32| { + line.and_then(|line| line.checked_sub(first)) + .filter(|&offset| offset < len) + }; + let (mut o, mut n) = (0, 0); + for row in rows { + let ro = offset(row.old, at.old_line, old_len); + let rn = offset(row.new, at.new_line, new_len); + if ro.is_some_and(|i| i != o) || rn.is_some_and(|i| i != n) { + return false; + } + steps.push(match (ro, rn) { + (Some(a), Some(b)) => Step::Paired(a, b), + (Some(a), None) => Step::Removed(a), + (None, Some(b)) => Step::Added(b), + (None, None) => continue, + }); + o += u32::from(ro.is_some()); + n += u32::from(rn.is_some()); + } + o == old_len && n == new_len +} + +/// The k-th removed line beside the k-th added line, then the rest. +fn positional_steps((old_len, new_len): (u32, u32), steps: &mut Vec) { + let paired = old_len.min(new_len); + steps.extend((0..paired).map(|k| Step::Paired(k, k))); + steps.extend((paired..old_len).map(Step::Removed)); + steps.extend((paired..new_len).map(Step::Added)); +} + +fn word_spans(ranges: &[Range]) -> impl Iterator, Intensity)> + '_ { + ranges.iter().map(|range| { + ( + range.start as usize..range.end as usize, + Intensity::NovelWord, + ) + }) +} + +fn semantic( + spans: &[diffy_diff::SemanticSpan], +) -> impl Iterator, Intensity)> + '_ { + spans.iter().map(|span| { + ( + span.start as usize..span.end as usize, + span.intensity.into(), + ) + }) +} + +fn context_row( + text: &TextStore, + syntax: &[HighlightSpan], + store: u32, + old_line: u32, + new_line: u32, +) -> Row { + let line = text.display_line(store).unwrap_or(""); + let mut units = Utf16::new(line); + Row::Context { + old_line, + new_line, + text: line.to_owned(), + syntax: syntax_spans(text, syntax, store, &mut units), + } +} + +/// Highlights a whole side, if `options` asks and it is within the size cap. +fn highlights( + options: &WireOptions<'_>, + path: Option<&str>, + text: &TextStore, +) -> Vec { + let source = text.as_str(); + match (options.syntax, path.and_then(LanguageId::from_path)) { + (Some(grammars), Some(language)) + if !source.is_empty() && source.len() <= options.syntax_max_bytes => + { + diffy_syntax::highlight(grammars, &language, source) + } + _ => Vec::new(), + } +} + +/// The spans of whole-text `syntax` on store line `store`, clipped to its +/// display text and converted to UTF-16 offsets within it. +fn syntax_spans( + text: &TextStore, + syntax: &[HighlightSpan], + store: u32, + units: &mut Utf16<'_>, +) -> Vec { + let Some(line) = text.display_range(store) else { + return Vec::new(); + }; + if syntax.is_empty() || line.is_empty() { + return Vec::new(); + } + // Spans are sorted and disjoint, so their ends ascend too. + let first = syntax.partition_point(|span| span.range().end <= line.start); + syntax[first..] + .iter() + .take_while(|span| (span.offset as usize) < line.end) + .filter(|span| span.kind != HighlightKind::Normal) + .filter_map(|span| { + let range = span.range(); + let start = range.start.max(line.start) - line.start; + let end = range.end.min(line.end) - line.start; + let (start, end) = units.range(start..end)?; + Some(SyntaxSpan { + start, + end, + class: Cow::Borrowed(span.kind.name()), + }) + }) + .collect() +} + +/// Converts byte offsets within a line to UTF-16 offsets, walking forward +/// from the previous offset; cheap for the ascending offsets of sorted +/// spans, and free for ASCII lines. +struct Utf16<'t> { + text: &'t str, + ascii: bool, + byte: usize, + unit: u32, +} + +impl<'t> Utf16<'t> { + fn new(text: &'t str) -> Self { + Self { + text, + ascii: text.is_ascii(), + byte: 0, + unit: 0, + } + } + + /// The UTF-16 offset of byte `at`, clamped to the text and moved back + /// to a char boundary. + fn at(&mut self, at: usize) -> u32 { + let mut at = at.min(self.text.len()); + if self.ascii { + return at as u32; + } + while !self.text.is_char_boundary(at) { + at -= 1; + } + if at < self.byte { + self.byte = 0; + self.unit = 0; + } + self.unit += self.text[self.byte..at].encode_utf16().count() as u32; + self.byte = at; + self.unit + } + + /// A byte range in UTF-16 offsets; `None` when it is empty. + fn range(&mut self, range: Range) -> Option<(u32, u32)> { + let start = self.at(range.start); + let end = self.at(range.end); + (start < end).then_some((start, end)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/diffy-compare/src/wire/tests.rs b/crates/diffy-compare/src/wire/tests.rs new file mode 100644 index 00000000..f9ae0d16 --- /dev/null +++ b/crates/diffy-compare/src/wire/tests.rs @@ -0,0 +1,934 @@ +use std::sync::Arc; + +use diffy_diff::{ + AlignedRow, ChangeIntensity, DocumentBuilder, FileMeta, FileStatus, ItemChange, SemanticSpan, + Side as DocSide, SourceCoverage, TextStore, diff_texts, +}; + +use super::*; +use crate::{FileDiff, FileInput}; + +const PATH: &str = "src/lib.rs"; + +fn input(old: &str, new: &str) -> FileInput { + FileInput { + status: FileStatus::Modified, + old_path: Some(PATH.into()), + new_path: Some(PATH.into()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + } +} + +fn line_diff(old: &str, new: &str) -> (FileInput, FileDiff) { + let document = diff_texts(Some(PATH), Some(PATH), Some(old), Some(new), 3); + let diff = FileDiff { + document, + fallback: Some(crate::Fallback::NoGrammar), + }; + (input(old, new), diff) +} + +fn numbered(lines: u32) -> Vec { + (1..=lines).map(|i| format!("line {i}")).collect() +} + +fn text(lines: &[String]) -> String { + let mut out = lines.join("\n"); + out.push('\n'); + out +} + +fn meta() -> FileMeta { + FileMeta { + old_path: Some(Arc::from(PATH)), + new_path: Some(Arc::from(PATH)), + status: FileStatus::Modified, + ..FileMeta::default() + } +} + +/// The text a JavaScript client sees at UTF-16 range `start..end`. +fn utf16_slice(text: &str, start: u32, end: u32) -> String { + let units: Vec = text.encode_utf16().collect(); + String::from_utf16_lossy(&units[start as usize..end as usize]) +} + +fn context(old_line: u32, new_line: u32, text: &str) -> Row { + Row::Context { + old_line, + new_line, + text: text.into(), + syntax: Vec::new(), + } +} + +fn span(start: u32, end: u32, intensity: Intensity) -> ChangeSpan { + ChangeSpan { + start, + end, + intensity, + } +} + +fn modified(row: &Row) -> (&Line, &Line) { + match row { + Row::Modified { old, new } => (old, new), + other => panic!("expected a modified row, got {other:?}"), + } +} + +#[test] +fn line_diff_projects_hunks_gaps_and_word_changes() { + let old = numbered(30); + let mut new = old.clone(); + new[14] = "line fifteen".into(); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let file = file(&input, &diff, &WireOptions::default()); + + assert_eq!(file.status, Status::Modified); + assert_eq!(file.language.as_deref(), None, "no_grammar hides the tag"); + assert_eq!((file.additions, file.deletions), (1, 1)); + assert_eq!(file.hunks.len(), 1); + let hunk = &file.hunks[0]; + assert_eq!( + (hunk.old_start, hunk.old_len, hunk.new_start, hunk.new_len), + (12, 7, 12, 7) + ); + assert_eq!(hunk.header, None); + assert_eq!( + hunk.gap_before, + Some(Gap { + old_start: 1, + new_start: 1, + len: 11 + }) + ); + assert_eq!( + file.gap_after, + Some(Gap { + old_start: 19, + new_start: 19, + len: 12 + }) + ); + assert_eq!(hunk.rows.len(), 7); + assert_eq!(hunk.rows[0], context(12, 12, "line 12")); + assert_eq!(hunk.rows[6], context(18, 18, "line 18")); + let (old_line, new_line) = modified(&hunk.rows[3]); + assert_eq!((old_line.line, old_line.text.as_str()), (15, "line 15")); + assert_eq!( + (new_line.line, new_line.text.as_str()), + (15, "line fifteen") + ); + assert_eq!(old_line.changes, [span(5, 7, Intensity::NovelWord)]); + assert_eq!(new_line.changes, [span(5, 12, Intensity::NovelWord)]); +} + +#[test] +fn documents_round_trip_through_json_and_omit_empty_fields() { + let old = numbered(8); + let mut new = old.clone(); + new[3] = "line four".into(); + new.push("line 9".into()); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let doc = document( + std::slice::from_ref(&input), + std::slice::from_ref(&diff), + &WireOptions::default(), + ); + assert_eq!(doc.version, FORMAT_VERSION); + + let json = serde_json::to_string(&doc).expect("serializes"); + let back: Document = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(back, doc); + + let value: serde_json::Value = serde_json::from_str(&json).expect("parses"); + let file = &value["files"][0]; + assert_eq!(value["version"], 1); + assert_eq!(file["status"], "modified"); + assert_eq!( + file["fallback"], + serde_json::json!({ "kind": "no_grammar", "message": "no grammar for this file" }) + ); + for absent in [ + "binary", + "language", + "outline", + "moves", + "renames", + "gap_after", + "old_missing_newline", + ] { + assert!(file.get(absent).is_none(), "{absent} should be omitted"); + } + let rows = &file["hunks"][0]["rows"]; + assert_eq!( + rows[0], + serde_json::json!({ "kind": "context", "old_line": 1, "new_line": 1, "text": "line 1" }) + ); + assert_eq!(rows[3]["kind"], "modified"); + assert_eq!( + rows[3]["old"]["changes"][0], + serde_json::json!({ "start": 5, "end": 6, "intensity": "novel_word" }) + ); + assert!(rows[3]["old"].get("syntax").is_none()); + let last = &rows[rows.as_array().expect("rows").len() - 1]; + assert_eq!( + *last, + serde_json::json!({ "kind": "added", "new": { "line": 9, "text": "line 9" } }) + ); +} + +#[test] +fn span_offsets_count_utf16_units() { + let old = "let s = \"héllo 😀 world\";\n"; + let new = "let s = \"héllo 😀 there\";\n"; + let (input, diff) = line_diff(old, new); + let file = file(&input, &diff, &WireOptions::default()); + let (old_line, new_line) = modified(&file.hunks[0].rows[0]); + let [old_span] = old_line.changes[..] else { + panic!("one old span: {:?}", old_line.changes); + }; + let [new_span] = new_line.changes[..] else { + panic!("one new span: {:?}", new_line.changes); + }; + // `é` is one unit and `😀` two, so "world" starts at 18, not byte 21. + assert_eq!((old_span.start, old_span.end), (18, 23)); + assert_eq!( + utf16_slice(&old_line.text, old_span.start, old_span.end), + "world" + ); + assert_eq!( + utf16_slice(&new_line.text, new_span.start, new_span.end), + "there" + ); +} + +#[test] +fn invalid_utf8_lines_show_replacement_characters() { + let old = b"caf\xe9 au lait\n".to_vec(); + let new = b"caf\xe9 au chocolat\n".to_vec(); + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::from_bytes(&old), + TextStore::from_bytes(&new), + ); + file_builder.hunk(0, 0).change(1, 1); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let input = FileInput { + old, + new, + ..input("", "") + }; + let projected = file(&input, &diff, &WireOptions::default()); + let (old_line, new_line) = modified(&projected.hunks[0].rows[0]); + assert_eq!(old_line.text, "caf\u{FFFD} au lait"); + assert_eq!(old_line.changes, [span(8, 12, Intensity::NovelWord)]); + assert_eq!( + utf16_slice( + &new_line.text, + new_line.changes[0].start, + new_line.changes[0].end + ), + "chocolat" + ); +} + +/// `render` renamed to `render_preview`; an argument changed after multibyte +/// text. +fn semantic_diff() -> (FileInput, FileDiff) { + let old = "fn keep() {}\nfn render() {\n draw(\"é😀\", 1);\n}\n"; + let new = "fn keep() {}\nfn render_preview() {\n draw(\"é😀\", 2);\n}\n"; + let (old_store, new_store) = (TextStore::new(old), TextStore::new(new)); + let at = |store: &TextStore, line: u32, needle: &str| { + let text = store.display_line(line).expect("line"); + let start = text.find(needle).expect("needle") as u32; + (start, start + needle.len() as u32) + }; + let semantic = |(start, end), intensity| SemanticSpan { + start, + end, + intensity, + }; + let old_spans = [ + semantic(at(&old_store, 1, "render"), ChangeIntensity::Renamed), + semantic(at(&old_store, 2, "1"), ChangeIntensity::NovelWord), + ]; + let new_spans = [ + semantic( + at(&new_store, 1, "render_preview"), + ChangeIntensity::Renamed, + ), + semantic(at(&new_store, 2, "2"), ChangeIntensity::NovelWord), + ]; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file(meta(), SourceCoverage::Full, old_store, new_store); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 2) + .context(1); + for line in 0..4 { + file_builder.row(AlignedRow { + old: Some(line), + new: Some(line), + }); + } + for (line, (old_span, new_span)) in (1..).zip(old_spans.iter().zip(&new_spans)) { + file_builder.spans(DocSide::Old, line, std::slice::from_ref(old_span)); + file_builder.spans(DocSide::New, line, std::slice::from_ref(new_span)); + } + file_builder + .outline(vec![diffy_diff::OutlineItem { + side: DocSide::New, + lines: 1..4, + kind: Arc::from("function_item"), + name: Arc::from("render_preview"), + change: ItemChange::Edited(2), + }]) + .renames(vec![diffy_diff::Rename { + old: Arc::from("render"), + new: Arc::from("render_preview"), + uses: 1, + }]) + .moves(vec![diffy_diff::Move { + old_line: 0, + old_lines: 1, + new_line: 0, + lines: 1, + from: None, + into: Some(Arc::from("render_preview")), + item: Some(Arc::from("keep")), + }]); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + (input(old, new), diff) +} + +#[test] +fn semantic_documents_keep_spans_outline_moves_and_renames() { + let (input, diff) = semantic_diff(); + let file = file(&input, &diff, &WireOptions::default()); + assert_eq!(file.language.as_deref(), Some("rs")); + assert_eq!(file.fallback, None); + assert_eq!( + file.outline, + [OutlineItem { + side: Side::New, + line: 2, + end_line: Some(4), + kind: "function_item".into(), + name: "render_preview".into(), + change: ItemChangeKind::Edited, + edits: Some(2), + }] + ); + assert_eq!( + file.renames, + [Rename { + old: "render".into(), + new: "render_preview".into(), + uses: 1, + }] + ); + assert_eq!( + file.moves, + [Move { + old_start: 1, + old_len: 1, + new_start: 1, + new_len: 1, + from: None, + into: Some("render_preview".into()), + item: Some("keep".into()), + }] + ); + + let hunk = &file.hunks[0]; + assert_eq!(hunk.header.as_deref(), Some("render_preview")); + assert_eq!((hunk.gap_before, file.gap_after), (None, None)); + assert_eq!(hunk.rows.len(), 4); + assert_eq!(hunk.rows[0], context(1, 1, "fn keep() {}")); + assert_eq!(hunk.rows[3], context(4, 4, "}")); + let (old_line, new_line) = modified(&hunk.rows[1]); + assert_eq!(old_line.changes, [span(3, 9, Intensity::Renamed)]); + assert_eq!(new_line.changes, [span(3, 17, Intensity::Renamed)]); + let (old_line, new_line) = modified(&hunk.rows[2]); + // Bytes 19..20, after `é` (2 bytes, 1 unit) and `😀` (4 bytes, 2 units). + assert_eq!(old_line.changes, [span(16, 17, Intensity::NovelWord)]); + assert_eq!(new_line.changes, [span(16, 17, Intensity::NovelWord)]); + assert_eq!(utf16_slice(&new_line.text, 16, 17), "2"); + + let json = serde_json::to_string(&file).expect("serializes"); + assert_eq!( + serde_json::from_str::(&json).expect("deserializes"), + file + ); + assert!(json.contains(r#""change":"edited","edits":2"#), "{json}"); +} + +#[test] +fn hunk_header_needs_an_item_containing_the_change() { + let (input, diff) = semantic_diff(); + let mut document = DocumentBuilder::new(); + let source = diff.document.text(0, DocSide::Old).clone(); + let target = diff.document.text(0, DocSide::New).clone(); + let mut file_builder = document.file(meta(), SourceCoverage::Full, source, target); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 2) + .context(1) + .outline(vec![diffy_diff::OutlineItem { + side: DocSide::Old, + lines: 3..4, + kind: Arc::from("function_item"), + name: Arc::from("later"), + change: ItemChange::Removed, + }]); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: document.build(), + fallback: None, + }; + let file = file(&input, &diff, &WireOptions::default()); + assert_eq!(file.hunks[0].header, None); +} + +#[test] +fn hunk_headers_name_the_innermost_containing_item() { + let item = |side, lines, name: &str| diffy_diff::OutlineItem { + side, + lines, + kind: Arc::from("function_item"), + name: Arc::from(name), + change: ItemChange::Edited(1), + }; + let header = |outline: Vec, (old_len, new_len): (u32, u32)| { + let side = |len, line| match len { + 0 => "zero\none\nthree\n".to_owned(), + _ => format!("zero\none\n{line}\nthree\n"), + }; + let (old, new) = (side(old_len, "old"), side(new_len, "new")); + let mut document = DocumentBuilder::new(); + let mut file_builder = document.file( + meta(), + SourceCoverage::Full, + TextStore::new(old.as_str()), + TextStore::new(new.as_str()), + ); + file_builder + .structural(true) + .hunk(0, 0) + .context(2) + .change(old_len, new_len) + .context(1) + .outline(outline); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: document.build(), + fallback: None, + }; + file(&input(&old, &new), &diff, &WireOptions::default()).hunks[0] + .header + .clone() + }; + // The change is on line 3 (zero-based 2) of each side. + let nested = || { + vec![ + item(DocSide::New, 0..4, "outer"), + item(DocSide::New, 1..4, "inner"), + item(DocSide::Old, 2..3, "removed"), + ] + }; + assert_eq!(header(nested(), (1, 1)).as_deref(), Some("inner")); + // Only the old side has lines, so its items come first. + assert_eq!(header(nested(), (1, 0)).as_deref(), Some("removed")); + // Without an old item, the new item must surround the gap on both sides. + let new_only = || { + vec![ + item(DocSide::New, 0..4, "outer"), + item(DocSide::New, 2..4, "after"), + ] + }; + assert_eq!(header(new_only(), (1, 0)).as_deref(), Some("outer")); + assert_eq!( + header(vec![item(DocSide::New, 0..2, "before")], (1, 0)), + None + ); +} + +#[test] +fn aligned_rows_pair_lines_where_the_semantic_diff_put_them() { + let old = "keep\none\ntwo\nend\n"; + let new = "keep\none!\ninserted\ntwo!\nend\n"; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::new(old), + TextStore::new(new), + ); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 3) + .context(1); + for (old, new) in [ + (Some(0), Some(0)), + (Some(1), Some(1)), + (None, Some(2)), + (Some(2), Some(3)), + (Some(3), Some(4)), + ] { + file_builder.row(AlignedRow { old, new }); + } + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let rows = &file(&input(old, new), &diff, &WireOptions::default()).hunks[0].rows; + let shape: Vec<(&str, Option, Option)> = rows + .iter() + .map(|row| match row { + Row::Context { + old_line, new_line, .. + } => ("context", Some(*old_line), Some(*new_line)), + Row::Removed { old } => ("removed", Some(old.line), None), + Row::Added { new } => ("added", None, Some(new.line)), + Row::Modified { old, new } => ("modified", Some(old.line), Some(new.line)), + }) + .collect(); + assert_eq!( + shape, + [ + ("context", Some(1), Some(1)), + ("modified", Some(2), Some(2)), + ("added", None, Some(3)), + ("modified", Some(3), Some(4)), + ("context", Some(4), Some(5)), + ] + ); + let (_, new_line) = modified(&rows[1]); + assert_eq!(new_line.changes, [span(3, 4, Intensity::NovelWord)]); +} + +#[test] +fn gaps_track_each_side_and_expand_into_context_rows() { + let old = numbered(30); + let mut new = old.clone(); + new.splice(10..10, ["new a".to_string(), "new b".to_string()]); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let file = file(&input, &diff, &WireOptions::default()); + let hunk = &file.hunks[0]; + assert_eq!( + (hunk.old_start, hunk.old_len, hunk.new_start, hunk.new_len), + (8, 6, 8, 8) + ); + let before = hunk.gap_before.expect("gap before"); + assert_eq!( + before, + Gap { + old_start: 1, + new_start: 1, + len: 7 + } + ); + let after = file.gap_after.expect("gap after"); + assert_eq!( + after, + Gap { + old_start: 14, + new_start: 16, + len: 17 + } + ); + + let options = WireOptions::default(); + let rows = expand(&input, &after, &options); + assert_eq!(rows.len(), 17); + assert_eq!(rows[0], context(14, 16, "line 14")); + assert_eq!(rows[16], context(30, 32, "line 30")); + + let tail = Gap { + old_start: 5, + new_start: 5, + len: 3, + }; + let rows = expand(&input, &tail, &options); + assert_eq!( + rows, + [ + context(5, 5, "line 5"), + context(6, 6, "line 6"), + context(7, 7, "line 7"), + ] + ); + + let past = Gap { + old_start: 29, + new_start: 31, + len: 10, + }; + assert_eq!(expand(&input, &past, &options).len(), 2); + // Stale: the sides differ. + let stale = Gap { + old_start: 14, + new_start: 14, + len: 3, + }; + assert!(expand(&input, &stale, &options).is_empty()); + let zero = Gap { + old_start: 0, + new_start: 0, + len: 3, + }; + assert!(expand(&input, &zero, &options).is_empty()); +} + +#[test] +fn unchanged_files_are_one_gap_and_added_files_have_none() { + let lines = text(&numbered(5)); + let (input, diff) = line_diff(&lines, &lines); + let file = file(&input, &diff, &WireOptions::default()); + assert!(file.hunks.is_empty()); + assert_eq!( + file.gap_after, + Some(Gap { + old_start: 1, + new_start: 1, + len: 5 + }) + ); + + let document = diff_texts(None, Some(PATH), None, Some("a\nb"), 3); + let input = FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some(PATH.into()), + old: Vec::new(), + new: b"a\nb".to_vec(), + }; + let diff = FileDiff { + document, + fallback: None, + }; + let file = super::file(&input, &diff, &WireOptions::default()); + assert_eq!(file.status, Status::Added); + assert!(file.new_missing_newline); + assert_eq!(file.gap_after, None); + let hunk = &file.hunks[0]; + assert_eq!(hunk.gap_before, None); + assert!(matches!( + &hunk.rows[..], + [Row::Added { .. }, Row::Added { .. }] + )); +} + +#[test] +fn binary_files_have_no_rows() { + let input = FileInput { + old: b"\0old".to_vec(), + new: b"\0new".to_vec(), + ..input("", "") + }; + let diff = FileDiff { + document: diff_texts(Some(PATH), Some(PATH), Some(""), Some(""), 3), + fallback: None, + }; + let file = file(&input, &diff, &WireOptions::default()); + assert!(file.binary); + assert!(file.hunks.is_empty() && file.gap_after.is_none()); + let json = serde_json::to_value(&file).expect("serializes"); + assert_eq!(json["binary"], true); + let gap = Gap { + old_start: 1, + new_start: 1, + len: 1, + }; + assert!(expand(&input, &gap, &WireOptions::default()).is_empty()); +} + +#[test] +fn fallbacks_serialize_as_stable_kinds() { + let cases = [ + ( + crate::Fallback::Semantic(diffy_semantic::Fallback::Timeout), + serde_json::json!({ + "kind": "semantic", + "reason": "timeout", + "message": "exceeded the semantic time budget", + }), + ), + ( + crate::Fallback::Semantic(diffy_semantic::Fallback::TooLarge(5)), + serde_json::json!({ + "kind": "semantic", + "reason": "too_large", + "message": "5 bytes exceeds the semantic byte limit", + }), + ), + ( + crate::Fallback::GrammarUnavailable("zig".into()), + serde_json::json!({ + "kind": "grammar_unavailable", + "message": "zig grammar unavailable", + }), + ), + ( + crate::Fallback::RequestBudget, + serde_json::json!({ + "kind": "request_budget", + "message": "request time budget exhausted", + }), + ), + ]; + for (fallback, expected) in cases { + let json = serde_json::to_value(Fallback::from(&fallback)).expect("serializes"); + assert_eq!(json, expected); + } +} + +/// Rust source whose `draw` call changes inside a method. +fn rust_change() -> (FileInput, FileDiff) { + let old = "fn helper() {}\n\nimpl Parser {\n fn render_preview(&self) {\n let s = \"é😀\";\n draw(1);\n }\n}\n"; + let new = old.replace("draw(1)", "draw(2)"); + line_diff(old, &new) +} + +#[test] +fn syntax_spans_are_classes_sliced_per_line_in_utf16() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let (input, diff) = rust_change(); + let options = WireOptions { + syntax: Some(&store), + ..WireOptions::default() + }; + let file = file(&input, &diff, &options); + let hunk = &file.hunks[0]; + assert_eq!(hunk.header.as_deref(), Some("fn render_preview")); + let Row::Context { text, syntax, .. } = &hunk.rows[2] else { + panic!("context row: {:?}", hunk.rows[2]); + }; + assert_eq!(text, " let s = \"é😀\";"); + let class_of = |slice: &str| { + syntax + .iter() + .find(|span| utf16_slice(text, span.start, span.end) == slice) + .map(|span| &*span.class) + }; + assert_eq!(class_of("let"), Some("keyword"), "{syntax:?}"); + assert_eq!(class_of("\"é😀\""), Some("string"), "{syntax:?}"); + let (old_line, _) = modified(&hunk.rows[3]); + let draw = old_line.syntax.first().expect("draw is highlighted"); + assert_eq!(utf16_slice(&old_line.text, draw.start, draw.end), "draw"); + assert_eq!(draw.class, "function"); + + let gap = hunk.gap_before.expect("gap before"); + let rows = expand(&input, &gap, &options); + let Row::Context { syntax, .. } = &rows[0] else { + panic!("context row: {:?}", rows[0]); + }; + assert!(syntax.iter().any(|span| span.class == "keyword")); + + let capped = WireOptions { + syntax: Some(&store), + syntax_max_bytes: 8, + ..WireOptions::default() + }; + let file = super::file(&input, &diff, &capped); + assert_eq!(file.hunks[0].header, None); + assert!(file.hunks[0].rows.iter().all(|row| match row { + Row::Context { syntax, .. } => syntax.is_empty(), + Row::Removed { old: line } | Row::Added { new: line } => line.syntax.is_empty(), + Row::Modified { old, new } => old.syntax.is_empty() && new.syntax.is_empty(), + })); +} + +#[test] +fn the_spec_example_matches_the_types() { + let spec = include_str!("../../FORMAT.md"); + let example = spec + .split("```json\n") + .nth(1) + .and_then(|rest| rest.split("```").next()) + .expect("FORMAT.md has a JSON example"); + let value: serde_json::Value = serde_json::from_str(example).expect("example parses"); + let document: Document = serde_json::from_value(value.clone()).expect("example is a Document"); + assert_eq!(document.version, FORMAT_VERSION); + // Round-trips exactly: no unknown or misspelled fields. + assert_eq!(serde_json::to_value(&document).expect("serializes"), value); +} + +#[test] +fn aligned_rows_that_miss_lines_fall_back_to_positional_pairs() { + let old = "one\ntwo\n"; + let new = "one!\ntwo!\n"; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::new(old), + TextStore::new(new), + ); + file_builder.structural(true).hunk(0, 0).change(2, 2); + // Only the first pair is aligned. + file_builder.row(AlignedRow { + old: Some(0), + new: Some(0), + }); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let rows = &file(&input(old, new), &diff, &WireOptions::default()).hunks[0].rows; + assert_eq!(rows.len(), 2, "{rows:?}"); + for (row, number) in rows.iter().zip(1..) { + let (old_line, new_line) = modified(row); + assert_eq!((old_line.line, new_line.line), (number, number)); + } +} + +#[test] +fn files_projected_past_the_deadline_skip_syntax() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let (input, diff) = rust_change(); + let options = WireOptions { + syntax: Some(&store), + deadline: Some(std::time::Instant::now()), + ..WireOptions::default() + }; + let doc = document(&[input.clone(), input], &[diff.clone(), diff], &options); + for file in &doc.files { + assert!(file.hunks[0].rows.iter().all(|row| match row { + Row::Context { syntax, .. } => syntax.is_empty(), + Row::Removed { old: line } | Row::Added { new: line } => line.syntax.is_empty(), + Row::Modified { old, new } => old.syntax.is_empty() && new.syntax.is_empty(), + })); + assert!(!file.hunks.is_empty()); + } +} + +/// Hunk headers of a semantic diff without syntax, so from the outline alone; +/// `None` without the pack. +fn outline_headers(language: &str, path: &str, old: &str, new: &str) -> Option> { + let store = diffy_syntax::testing::store_with(language)?; + let input = FileInput { + status: FileStatus::Modified, + old_path: Some(path.into()), + new_path: Some(path.into()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + }; + let diff = crate::diff_file(&input, &crate::Options::new(&store)).expect("diffs"); + assert_eq!(diff.fallback, None); + let file = file(&input, &diff, &WireOptions::default()); + Some( + file.hunks + .into_iter() + .map(|hunk| hunk.header.unwrap_or_default()) + .collect(), + ) +} + +#[test] +fn hunk_headers_name_only_items_containing_the_change() { + let old = "def first(): + a = 1 + b = 2 + c = 3 + return a + + +x = 1 +y = 2 +z = 3 +w = 4 +v = 5 +u = 6 +t = 7 +s = 8 +r = 9 + + +def second(): + p = 1 + q = 2 + return p +"; + let new = old + .replace("a = 1", "a = 10") + .replace("v = 5", "v = 50") + .replace("q = 2", "q = 20"); + let Some(headers) = outline_headers("python", "src/app.py", old, &new) else { + return; + }; + // The middle change is a top-level statement below `first`. + assert_eq!(headers, ["first", "", "second"]); +} + +#[test] +fn hunk_headers_pick_the_innermost_item_on_the_change_side() { + let old = "impl Parser { + fn render(&self) { + draw(1); + draw(2); + draw(3); + draw(4); + draw(5); + draw(6); + } + + fn other(&self) { + a(); + b(); + c(); + d(); + e(); + f(); + g(); + h(); + } + + fn stay(&self) { + one(); + two(); + three(); + four(); + } + + fn removed(&self) { + gone(); + } +} +"; + let new = old + .replace("draw(1)", "draw(10)") + .replace(" b();\n", "") + .replace("\n fn removed(&self) {\n gone();\n }\n", ""); + let Some(headers) = outline_headers("rust", "src/lib.rs", old, &new) else { + return; + }; + // Edited method, not its impl; a deletion inside an edited method (new + // side); a removed method (old side). + assert_eq!(headers, ["render", "other", "removed"]); +} diff --git a/crates/diffy-diff/Cargo.toml b/crates/diffy-diff/Cargo.toml index 687c8a5a..9982dc96 100644 --- a/crates/diffy-diff/Cargo.toml +++ b/crates/diffy-diff/Cargo.toml @@ -4,6 +4,7 @@ description = "Diff model for Diffy: unified patch parsing, line and word diffs, version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false # No UI, text shaping, or syntax dependencies: the diff view in diffy-ui diff --git a/crates/diffy-diff/LICENSE-APACHE b/crates/diffy-diff/LICENSE-APACHE new file mode 100644 index 00000000..9267b22b --- /dev/null +++ b/crates/diffy-diff/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Rohit (seatedro) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-diff/LICENSE-MIT b/crates/diffy-diff/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-diff/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-diff/src/semantic.rs b/crates/diffy-diff/src/semantic.rs index 054b9705..5ace8266 100644 --- a/crates/diffy-diff/src/semantic.rs +++ b/crates/diffy-diff/src/semantic.rs @@ -116,7 +116,8 @@ pub struct Rename { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OutlineItem { pub side: Side, - pub line: u32, + /// Zero-based lines the item spans on `side`; never empty. + pub lines: Range, pub kind: Arc, pub name: Arc, pub change: ItemChange, diff --git a/crates/diffy-semantic/Cargo.toml b/crates/diffy-semantic/Cargo.toml index 31718865..22ceb26d 100644 --- a/crates/diffy-semantic/Cargo.toml +++ b/crates/diffy-semantic/Cargo.toml @@ -4,6 +4,7 @@ description = "Structural diff of two sources parsed with tree-sitter, for Diffy version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false [dependencies] diff --git a/crates/diffy-semantic/LICENSE-APACHE b/crates/diffy-semantic/LICENSE-APACHE new file mode 100644 index 00000000..9267b22b --- /dev/null +++ b/crates/diffy-semantic/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Rohit (seatedro) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-semantic/LICENSE-MIT b/crates/diffy-semantic/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-semantic/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-semantic/src/align.rs b/crates/diffy-semantic/src/align.rs index 58fb0881..62798d7d 100644 --- a/crates/diffy-semantic/src/align.rs +++ b/crates/diffy-semantic/src/align.rs @@ -680,7 +680,7 @@ fn outline(sides: &[Side<'_, '_>; 2], matching: &Matching) -> Vec { }; out.push(OutlineItem { side: which, - line: item.lines.start, + lines: item.lines.clone(), kind: item.kind.into(), name: side.tree.text(item.name.clone()).into(), change, diff --git a/crates/diffy-syntax/Cargo.toml b/crates/diffy-syntax/Cargo.toml index 7c480ecc..02d68d43 100644 --- a/crates/diffy-syntax/Cargo.toml +++ b/crates/diffy-syntax/Cargo.toml @@ -1,19 +1,21 @@ [package] name = "diffy-syntax" -description = "Tree-sitter syntax highlighting for Diffy, with grammars loaded at runtime from signed packs." +description = "Tree-sitter syntax highlighting for Diffy, with grammars loaded at runtime from signed packs or compiled in." version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false -# No grammar is compiled in. Grammars arrive as packs (a shared library -# plus queries and a manifest) from a local directory the app names, or -# from a signed index the app configures; see the crate docs and -# PACKS.md. Without any feature this crate has no +# No grammar is compiled in unless `bundled` is on. Grammars arrive as +# packs (a shared library plus queries and a manifest) from a local +# directory the app names, or from a signed index the app configures; see +# the crate docs and PACKS.md. Without any feature this crate has no # dependencies and every lookup falls back to plain text. [dependencies] tree-sitter = { version = "0.26", optional = true } tree-sitter-language = { version = "0.1", optional = true } +tracing = { version = "0.1.44", optional = true } libloading = { version = "0.8", optional = true } # SHA-256 of pack files, and Ed25519 for the index with `download`. The # same ring that ureq's rustls backend builds. @@ -21,7 +23,6 @@ ring = { version = "0.17", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } serde_json = { version = "1.0", optional = true } thiserror = { version = "2", optional = true } -tracing = { version = "0.1.44", optional = true } # Resumable, SHA-checked downloads, the trusted key type, and the HTTP # client (ureq with rustls), shared with the updater. quark-update = { workspace = true, optional = true } @@ -29,19 +30,21 @@ ureq = { version = "3", default-features = false, features = ["rustls"], optiona [features] default = [] -# The tree-sitter runtime and pack loading from local directories. +# Tree-sitter runtime (highlighting, injections, scopes, symbols); no grammars. +runtime = ["dep:tree-sitter", "dep:tree-sitter-language", "dep:tracing"] +# `runtime` plus pack loading from local directories. engine = [ - "dep:tree-sitter", - "dep:tree-sitter-language", + "runtime", "dep:libloading", "dep:ring", "dep:serde", "dep:serde_json", "dep:thiserror", - "dep:tracing", ] # Downloading packs on demand from a signed index. download = ["engine", "dep:quark-update", "dep:ureq"] +# Grammars compiled in from $DIFFY_SYNTAX_BUNDLE (`syntax-pack bundle`). +bundled = ["runtime"] [dev-dependencies] tempfile = "3" diff --git a/crates/diffy-syntax/LICENSE-APACHE b/crates/diffy-syntax/LICENSE-APACHE new file mode 100644 index 00000000..9267b22b --- /dev/null +++ b/crates/diffy-syntax/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Rohit (seatedro) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-syntax/LICENSE-MIT b/crates/diffy-syntax/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-syntax/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-syntax/PACKS.md b/crates/diffy-syntax/PACKS.md index 7c528911..d61e1843 100644 --- a/crates/diffy-syntax/PACKS.md +++ b/crates/diffy-syntax/PACKS.md @@ -2,7 +2,8 @@ Load tree-sitter grammars at runtime, from packs, to highlight diffs. -- No grammar is compiled into the app. +- No grammar is compiled into the app; other builds can use + [bundled grammars](#bundled-grammars). - A pack is a shared library exporting the tree-sitter language function, plus its highlight query, an optional injection query, and a manifest. - A `GrammarStore` feeds the diff view; an app adapts it to Quark's @@ -16,6 +17,7 @@ Load tree-sitter grammars at runtime, from packs, to highlight diffs. |---|---|---|---| | Local pack roots | `StoreConfig::local_packs(root)` | `diffy-syntax/engine` | Unsigned; the app opts in | | Signed index | `StoreConfig::downloads(Downloads::new(app_id, url, &keys))` | `diffy-syntax/download` | Ed25519 signature over the index | +| Bundled grammars | `StoreConfig::bundled()` | `diffy-syntax/bundled` | Part of the binary | From [diff_demo.rs](../diffy-ui/examples/diff_demo.rs): @@ -26,8 +28,8 @@ let store = GrammarStore::new(StoreConfig::new().local_packs(root)); diff.enable_syntax(store); // DiffViewState ``` -- Search order: local roots (in the order added), then the download cache, - then the network. +- Search order: local roots (in the order added), then bundled grammars, + then the download cache, then the network. - No default root and no default index URL. - A pack root holds `//`, as `syntax-pack` writes it. @@ -66,12 +68,14 @@ A fence tag or file extension resolves once per store; the result is kept, so an unknown language costs one map lookup afterwards. 1. A local pack whose name, alias, or extension matches loads at once. -2. A pack in the cached signed index, with every file present and matching +2. With `StoreConfig::bundled`, a bundled grammar that matches loads at + once. +3. A pack in the cached signed index, with every file present and matching its SHA-256, loads at once. -3. Otherwise, with downloads configured, the language is pending. The fetch +4. Otherwise, with downloads configured, the language is pending. The fetch thread downloads the index (at most once per store, cached) and then the pack. -4. A language missing from the fresh index, or whose download fails, is +5. A language missing from the fresh index, or whose download fails, is unavailable until `GrammarStore::retry` (which refetches the index). - `HighlightWorker` answers a pending language with a plain result marked @@ -128,6 +132,38 @@ cargo run -p syntax-pack -- check # re-check built packs for thi - TypeScript and TSX also read JavaScript's queries from `DIR/javascript/`. +## Bundled grammars + +Compiled-in grammars, for builds without packs or downloads (such as a +headless server). The app does not use them. + +- Default set: C, C++, CSS, Go, HTML, JavaScript, JSON, Kotlin, Markdown + (with `markdown_inline`), Python, Rust, Swift, TOML, TSX, TypeScript, + YAML. +- `syntax-pack bundle` writes `//`: `lib/` (static + libraries), `queries/`, `bundle.tsv`, `THIRD_PARTY_NOTICES.md`. +- Build with `--features bundled` and `DIFFY_SYNTAX_BUNDLE` set to the + absolute ``. Unset, the bundle is empty and cargo warns. +- `bundled` needs only `runtime`: no `libloading`, `ring`, JSON manifests, + or `StoreConfig::local_packs`. Add `engine` to also load packs. +- Same tags, queries, ABI check, and `catalog()` entries as packs. +- `diffy_syntax::bundled::languages()` lists what a build has. +- Ship `THIRD_PARTY_NOTICES.md` with anything you distribute; the grammar + licenses require it. + +```sh +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" # default set, this host +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" rust go # just these +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" --target x86_64-apple-darwin +DIFFY_SYNTAX_BUNDLE="$PWD/target/syntax-bundle" cargo build -p my-server --features diffy-syntax/bundled +``` + +```rust +use diffy_syntax::{GrammarStore, StoreConfig}; + +let store = GrammarStore::new(StoreConfig::new().bundled()); +``` + Maintainer details (pinning, cross-target checks, publishing) are in [tools/syntax-pack/README.md](../../tools/syntax-pack/README.md). @@ -175,5 +211,7 @@ Loading a pack runs native code with the app's privileges. Full model: - **The cache** is not defended against anyone who can already write the user's files (including swapping a library between hash check and `dlopen`). +- **Bundled grammars** are part of the binary, built from the same pinned, + hashed sources; nothing is loaded at run time. - **Parsers** see untrusted text. A crash in a grammar's C code kills the process; a panic in highlighting is caught by the worker. diff --git a/crates/diffy-syntax/README.md b/crates/diffy-syntax/README.md index fd39c87f..09316d9b 100644 --- a/crates/diffy-syntax/README.md +++ b/crates/diffy-syntax/README.md @@ -1,14 +1,18 @@ # diffy-syntax Tree-sitter syntax highlighting for [Diffy](../../README.md). -No grammar is compiled in: each language is a pack (a shared library, its -queries, and a manifest) that a `GrammarStore` loads from local pack -directories the app trusts or, with the `download` feature, from a signed -index. Code renders plain until its grammar is available. +No grammar is compiled in by default: each language is a pack (a shared +library, its queries, and a manifest) that a `GrammarStore` loads from local +pack directories the app trusts or, with the `download` feature, from a +signed index. Code renders plain until its grammar is available. With +`bundled`, a fixed set is compiled in instead. -Features: `engine` (the tree-sitter runtime and local packs) and `download` -(fetching packs over HTTPS with Ed25519-verified indexes). Without `engine` -the crate has no dependencies and every lookup falls back to plain text. +Features: `runtime` (tree-sitter highlighting, injections, scopes, +symbols), `engine` (`runtime` plus local packs), `download` (`engine` plus +fetching packs over HTTPS with Ed25519-verified indexes), and `bundled` +(`runtime` plus grammars from `syntax-pack bundle`; no pack loading). +Without `runtime` the crate has no dependencies and every lookup falls back +to plain text. `highlight` runs synchronously; `HighlightWorker` runs on a background thread and drops requests a newer generation superseded, which suits code diff --git a/crates/diffy-syntax/build.rs b/crates/diffy-syntax/build.rs index 205268e9..6e69c085 100644 --- a/crates/diffy-syntax/build.rs +++ b/crates/diffy-syntax/build.rs @@ -1,3 +1,12 @@ +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +/// The directory `syntax-pack bundle --out` wrote. +const BUNDLE_VAR: &str = "DIFFY_SYNTAX_BUNDLE"; +/// First line of `//bundle.tsv`. +const HEADER: &str = "# diffy-syntax bundle 1"; +const QUERIES: [&str; 5] = ["highlights", "injections", "locals", "semantic", "tags"]; + // Packs are built per target triple, and Rust has no runtime constant for // the full triple (`std::env::consts` lacks the vendor and ABI parts that // tell gnu from musl, or msvc from gnu), so record the one this build is for. @@ -5,4 +14,163 @@ fn main() { let target = std::env::var("TARGET").expect("cargo sets TARGET for build scripts"); println!("cargo:rustc-env=DIFFY_SYNTAX_TARGET={target}"); println!("cargo:rerun-if-changed=build.rs"); + if std::env::var_os("CARGO_FEATURE_BUNDLED").is_some() + && let Err(error) = bundle(&target) + { + panic!("{BUNDLE_VAR}: {error}"); + } +} + +/// Writes `$OUT_DIR/bundled.rs` for `src/bundled.rs` and links the bundle. +fn bundle(target: &str) -> Result<(), String> { + println!("cargo:rerun-if-env-changed={BUNDLE_VAR}"); + let out_dir = std::env::var_os("OUT_DIR").ok_or("cargo did not set OUT_DIR")?; + let code = match std::env::var_os(BUNDLE_VAR).filter(|value| !value.is_empty()) { + None => { + println!( + "cargo:warning={BUNDLE_VAR} is not set, so the `bundled` feature compiles no \ + grammar in; see crates/diffy-syntax/PACKS.md" + ); + "pub(crate) static GRAMMARS: &[Bundled] = &[];\n".to_owned() + } + Some(root) => { + let root = PathBuf::from(root); + if !root.is_absolute() { + return Err(format!("{} is not an absolute path", root.display())); + } + generate(&root.join(target), target)? + } + }; + let out = Path::new(&out_dir).join("bundled.rs"); + std::fs::write(&out, code).map_err(|e| format!("{}: {e}", out.display())) +} + +/// The table for the bundle in `dir`. `bundle.tsv` columns: language, +/// symbol, ABI, library, aliases, extensions, source SHA-256 (lists +/// comma-separated, `-` when empty). Queries: `queries//.scm`. +fn generate(dir: &Path, target: &str) -> Result { + let manifest = dir.join("bundle.tsv"); + let text = std::fs::read_to_string(&manifest).map_err(|e| { + format!( + "{}: {e}; write it with `syntax-pack bundle --target {target}`", + manifest.display() + ) + })?; + println!("cargo:rerun-if-changed={}", manifest.display()); + let mut lines = text.lines(); + if lines.next() != Some(HEADER) { + return Err(format!("{} is not a version 1 bundle", manifest.display())); + } + let lib_dir = dir.join("lib"); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + let mut externs = String::new(); + let mut table = String::new(); + let lines = lines.filter(|line| !line.is_empty() && !line.starts_with('#')); + for (index, line) in lines.enumerate() { + let bad = |what: &str| format!("{}: {what} in {line:?}", manifest.display()); + let fields: Vec<&str> = line.split('\t').collect(); + let [language, symbol, abi, library, aliases, extensions, source] = fields[..] else { + return Err(bad("not 7 tab-separated fields")); + }; + let list = |field: &str| -> Result, String> { + if field == "-" { + return Ok(Vec::new()); + } + field + .split(',') + .map(|tag| { + is_tag(tag) + .then(|| format!("{tag:?}")) + .ok_or_else(|| bad(&format!("bad tag {tag:?}"))) + }) + .collect() + }; + let (aliases, extensions) = (list(aliases)?, list(extensions)?); + if !is_tag(language) { + return Err(bad("bad language")); + } + if !is_c_identifier(symbol) { + return Err(bad("bad symbol")); + } + let abi: u32 = abi.parse().map_err(|_| bad("bad ABI"))?; + if library.is_empty() + || !library + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + return Err(bad("bad library name")); + } + if source.len() != 64 || !source.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(bad("bad source hash")); + } + let archive = if target.contains("msvc") { + lib_dir.join(format!("{library}.lib")) + } else { + lib_dir.join(format!("lib{library}.a")) + }; + if !archive.is_file() { + return Err(format!("{} is missing", archive.display())); + } + println!("cargo:rerun-if-changed={}", archive.display()); + println!("cargo:rustc-link-lib=static={library}"); + + // Only the link name comes from the bundle; the Rust name is ours. + let _ = writeln!( + externs, + " #[link_name = {symbol:?}]\n fn grammar_{index}() -> *const ();" + ); + let mut queries = String::new(); + for name in QUERIES { + let path = dir + .join("queries") + .join(language) + .join(format!("{name}.scm")); + let value = if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + let path = path + .to_str() + .ok_or_else(|| format!("{} is not UTF-8", path.display()))?; + if name == "highlights" { + format!("include_str!({path:?})") + } else { + format!("Some(include_str!({path:?}))") + } + } else if name == "highlights" { + return Err(format!("{} is missing", path.display())); + } else { + "None".to_owned() + }; + let _ = writeln!(queries, " {name}: {value},"); + } + let _ = write!( + table, + " Bundled {{\n name: {language:?},\n aliases: &[{}],\n \ + extensions: &[{}],\n abi: {abi},\n language: grammar_{index},\n \ + source: {source:?},\n{queries} }},\n", + aliases.join(", "), + extensions.join(", "), + ); + } + Ok(format!( + "// Generated by build.rs from {:?}.\nunsafe extern \"C\" {{\n{externs}}}\n\n\ + pub(crate) static GRAMMARS: &[Bundled] = &[\n{table}];\n", + manifest + )) +} + +/// A fence tag or extension, as `LanguageId::from_fence` accepts them. +fn is_tag(value: &str) -> bool { + !value.is_empty() + && value.len() <= 32 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'#' | b'.' | b'_' | b'-')) +} + +fn is_c_identifier(value: &str) -> bool { + let mut bytes = value.bytes(); + bytes + .next() + .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') } diff --git a/crates/diffy-syntax/src/bundled.rs b/crates/diffy-syntax/src/bundled.rs new file mode 100644 index 00000000..029c379c --- /dev/null +++ b/crates/diffy-syntax/src/bundled.rs @@ -0,0 +1,44 @@ +//! Grammars compiled in with the `bundled` feature, from the +//! `syntax-pack bundle` output `DIFFY_SYNTAX_BUNDLE` names at build time +//! (empty without it). Stores use them only after +//! [`crate::StoreConfig::bundled`]. + +/// One compiled-in grammar, as `build.rs` generates it. +pub(crate) struct Bundled { + pub(crate) name: &'static str, + pub(crate) aliases: &'static [&'static str], + pub(crate) extensions: &'static [&'static str], + /// Declared tree-sitter ABI, checked against the linked grammar. + pub(crate) abi: u32, + pub(crate) language: unsafe extern "C" fn() -> *const (), + /// SHA-256 of the pinned sources it was built from. + pub(crate) source: &'static str, + pub(crate) highlights: &'static str, + pub(crate) injections: Option<&'static str>, + pub(crate) locals: Option<&'static str>, + pub(crate) semantic: Option<&'static str>, + pub(crate) tags: Option<&'static str>, +} + +impl Bundled { + /// Lookup keys: name, aliases, extensions. + pub(crate) fn tags(&self) -> impl Iterator { + std::iter::once(self.name) + .chain(self.aliases.iter().copied()) + .chain(self.extensions.iter().copied()) + } +} + +include!(concat!(env!("OUT_DIR"), "/bundled.rs")); + +/// Names of the compiled-in grammars; empty without `DIFFY_SYNTAX_BUNDLE`. +pub fn languages() -> impl ExactSizeIterator + Clone { + GRAMMARS.iter().map(|grammar| grammar.name) +} + +/// By name, alias, or extension, ignoring ASCII case. +pub(crate) fn find(tag: &str) -> Option<&'static Bundled> { + GRAMMARS + .iter() + .find(|grammar| grammar.tags().any(|known| known.eq_ignore_ascii_case(tag))) +} diff --git a/crates/diffy-syntax/src/engine.rs b/crates/diffy-syntax/src/engine.rs index 4ededefb..4f6cfa03 100644 --- a/crates/diffy-syntax/src/engine.rs +++ b/crates/diffy-syntax/src/engine.rs @@ -1,5 +1,6 @@ -//! Loading grammars from packs, parsing, and query evaluation, including -//! languages embedded in others (tree-sitter injections). +//! Loading grammars from packs or the bundle, parsing, and query +//! evaluation, including languages embedded in others (tree-sitter +//! injections). //! //! A [`Grammar`] is shared by every thread; parsers are not `Sync`, so each //! thread keeps its own per grammar. @@ -13,6 +14,7 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; use std::ops::ControlFlow; +#[cfg(feature = "engine")] use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -20,10 +22,42 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tree_sitter as ts; use tree_sitter::StreamingIterator; -use crate::pack::{PackError, PackManifest, supported_abi, verify_files}; +#[cfg(feature = "engine")] +use crate::pack::{PackError, PackManifest, verify_files}; use crate::store::Tag; use crate::{HighlightKind, HighlightSpan, LanguageId}; +/// Grammar ABI versions (tree-sitter `LANGUAGE_VERSION`) this runtime can +/// load. +pub fn supported_abi() -> std::ops::RangeInclusive { + ts::MIN_COMPATIBLE_LANGUAGE_VERSION as u32..=ts::LANGUAGE_VERSION as u32 +} + +/// Why a grammar did not compile; packs map it to `pack::PackError`. +#[derive(Debug)] +pub(crate) enum GrammarError { + Abi { abi: u32, min: u32, max: u32 }, + Query(String), + InjectionQuery(String), +} + +impl std::fmt::Display for GrammarError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Abi { abi, min, max } => { + write!( + f, + "grammar ABI {abi} is outside the supported {min}..={max}" + ) + } + Self::Query(error) => write!(f, "the highlight query does not compile: {error}"), + Self::InjectionQuery(error) => { + write!(f, "the injection query does not compile: {error}") + } + } + } +} + /// A loaded language: its tree-sitter grammar, compiled highlight query, /// and compiled injection query when its pack has one. pub(crate) struct Grammar { @@ -59,6 +93,7 @@ impl Grammar { /// The library stays loaded for the rest of the process: tree-sitter /// languages, parsers, and trees on other threads point into it, and /// nothing tracks when the last of them is gone. + #[cfg(feature = "engine")] pub(crate) fn load(dir: &Path, manifest: &PackManifest) -> Result { verify_files(dir, manifest)?; let highlights = std::fs::read_to_string(dir.join(&manifest.highlights.path))?; @@ -67,13 +102,24 @@ impl Grammar { None => Ok(None), }; let injections = read(&manifest.injections)?; - let grammar = Self::compile( - &dir.join(&manifest.library.path), - &manifest.symbol, - Some(manifest.abi), - &highlights, - injections.as_deref(), - )?; + let library_path = dir.join(&manifest.library.path); + // SAFETY: loading a library runs its initializers, and calling the + // symbol runs its code. The pack passed its SHA-256 check against a + // manifest that is either signed (downloaded packs) or in a + // directory the app opted into (local packs); see the threat model + // in the crate docs. + let language = unsafe { + let library = libloading::Library::new(&library_path) + .map_err(|e| PackError::Library(e.to_string()))?; + let library: &'static libloading::Library = Box::leak(Box::new(library)); + let mut name = manifest.symbol.as_bytes().to_vec(); + name.push(0); + let symbol = library + .get:: *const ()>(&name) + .map_err(|e| PackError::Library(e.to_string()))?; + tree_sitter_language::LanguageFn::from_raw(*symbol) + }; + let grammar = Self::compile(language, manifest.abi, &highlights, injections.as_deref())?; Ok(Self { name: Some(Arc::from(manifest.language.as_str())), locals: read(&manifest.locals)?.map(Arc::from), @@ -84,6 +130,28 @@ impl Grammar { }) } + #[cfg(feature = "bundled")] + pub(crate) fn bundled(bundled: &crate::bundled::Bundled) -> Result { + // SAFETY: a tree-sitter language function linked in from pinned, + // hashed sources by `syntax-pack bundle`. + let language = unsafe { tree_sitter_language::LanguageFn::from_raw(bundled.language) }; + let grammar = Self::compile( + language, + bundled.abi, + bundled.highlights, + bundled.injections, + )?; + Ok(Self { + name: Some(Arc::from(bundled.name)), + locals: bundled.locals.map(Arc::from), + semantic: bundled.semantic.map(Arc::from), + tags: bundled.tags.map(Arc::from), + // Keys tag caches, like a pack's library hash. + library: Some(Arc::from(bundled.source)), + ..grammar + }) + } + pub(crate) fn name(&self) -> Option<&str> { self.name.as_deref() } @@ -112,41 +180,26 @@ impl Grammar { self.tags.as_ref() } - pub(crate) fn compile( - library_path: &Path, - symbol: &str, - expected_abi: Option, + /// Checks the ABI against `expected_abi` and compiles the queries. + fn compile( + language: tree_sitter_language::LanguageFn, + expected_abi: u32, highlights: &str, injections: Option<&str>, - ) -> Result { - // SAFETY: loading a library runs its initializers, and calling the - // symbol runs its code. The pack passed its SHA-256 check against a - // manifest that is either signed (downloaded packs) or in a - // directory the app opted into (local packs); see the threat model - // in the crate docs. - let language = unsafe { - let library = libloading::Library::new(library_path) - .map_err(|e| PackError::Library(e.to_string()))?; - let library: &'static libloading::Library = Box::leak(Box::new(library)); - let mut name = symbol.as_bytes().to_vec(); - name.push(0); - let symbol = library - .get:: *const ()>(&name) - .map_err(|e| PackError::Library(e.to_string()))?; - ts::Language::new(tree_sitter_language::LanguageFn::from_raw(*symbol)) - }; - // The manifest's ABI was checked; this checks the library agrees. + ) -> Result { + let language = ts::Language::new(language); + // The declared ABI was checked; this checks the grammar agrees. let abi = language.abi_version() as u32; - if expected_abi.is_some_and(|expected| expected != abi) || !supported_abi().contains(&abi) { + if expected_abi != abi || !supported_abi().contains(&abi) { let range = supported_abi(); - return Err(PackError::Abi { + return Err(GrammarError::Abi { abi, min: *range.start(), max: *range.end(), }); } let query = - lenient_query(&language, highlights).map_err(|e| PackError::Query(e.to_string()))?; + lenient_query(&language, highlights).map_err(|e| GrammarError::Query(e.to_string()))?; let capture_kinds = query .capture_names() .iter() @@ -338,9 +391,9 @@ struct InjectionPattern { } impl Injections { - fn compile(language: &ts::Language, text: &str) -> Result { - let query = - lenient_query(language, text).map_err(|e| PackError::InjectionQuery(e.to_string()))?; + fn compile(language: &ts::Language, text: &str) -> Result { + let query = lenient_query(language, text) + .map_err(|e| GrammarError::InjectionQuery(e.to_string()))?; let patterns = (0..query.pattern_count()) .map(|index| { let mut pattern = InjectionPattern::default(); diff --git a/crates/diffy-syntax/src/lib.rs b/crates/diffy-syntax/src/lib.rs index 9d75798c..8074a1b2 100644 --- a/crates/diffy-syntax/src/lib.rs +++ b/crates/diffy-syntax/src/lib.rs @@ -1,12 +1,13 @@ //! Tree-sitter syntax highlighting for code blocks, with grammars loaded at //! runtime. //! -//! No grammar is compiled in. Each language arrives as a pack, a shared -//! library exporting the tree-sitter language function plus its queries -//! and a manifest (see [`pack`]), from a [`GrammarStore`] the app +//! No grammar is compiled in by default. Each language arrives as a pack, +//! a shared library exporting the tree-sitter language function plus its +//! queries and a manifest (see [`pack`]), from a [`GrammarStore`] the app //! configures: local pack directories it trusts, and with the `download` //! feature a signed index it downloads missing packs from. Until a -//! language's grammar is available its code renders plain. +//! language's grammar is available its code renders plain. The `bundled` +//! feature links a fixed set instead (see [`bundled`]). //! //! [`highlight`] runs synchronously; [`HighlightWorker`] runs it on a //! background thread and drops requests superseded by a newer generation @@ -24,9 +25,11 @@ //! an embedded grammar downloads, its region keeps the host's colors, and //! the worker sends a newer revision of the result when it arrives. //! -//! Features: `engine` (the tree-sitter runtime and local packs) and -//! `download` (fetching packs). Without `engine` every lookup is plain and -//! the crate has no dependencies. +//! Features: `runtime` (the tree-sitter runtime: highlighting, injections, +//! scopes, and symbols), `engine` (`runtime` plus loading local packs), +//! `download` (`engine` plus fetching packs), and `bundled` (`runtime` plus +//! grammars linked in, with no pack loading). Without `runtime` every +//! lookup is plain and the crate has no dependencies. //! //! # Threat model //! @@ -56,10 +59,21 @@ //! - Grammars parse untrusted text (chat output, diffs). A parser crash //! takes the process down; a panic in the highlighter is caught by the //! worker. Packs come from pinned, hashed grammar sources built in CI. - +//! - Bundled grammars are part of the binary, built from the same pinned, +//! hashed sources; nothing is loaded at run time. + +// `runtime` alone has no grammar source; a dependent enabling `engine` or +// `bundled` supplies one through feature unification. +#![cfg_attr( + all(feature = "runtime", not(any(feature = "engine", feature = "bundled"))), + allow(dead_code, unused_mut, unused_variables) +)] + +#[cfg(feature = "bundled")] +pub mod bundled; #[cfg(feature = "download")] mod download; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] mod engine; mod navigate; #[cfg(feature = "engine")] @@ -81,10 +95,10 @@ pub use navigate::{DefinitionLookup, SourcePos}; pub use quark_update::PublicKey; pub use scope::{ScopeResult, ScopeSpan, ScopeTable, ScopeWorker, scope_table}; pub use store::{Catalog, GrammarState, GrammarStatus, GrammarStore, LanguageStatus}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub use store::{GrammarQueries, Injector, StoreConfig}; pub use symbols::{Symbol, SymbolKind}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub use symbols::{Tagger, TaggerLookup}; pub use worker::{ HighlightFocus, HighlightPart, HighlightRequest, HighlightWorker, Highlighted, Priority, diff --git a/crates/diffy-syntax/src/navigate.rs b/crates/diffy-syntax/src/navigate.rs index d02f08a5..bb9c28f6 100644 --- a/crates/diffy-syntax/src/navigate.rs +++ b/crates/diffy-syntax/src/navigate.rs @@ -15,14 +15,14 @@ pub struct DefinitionLookup { pub symbols: Vec, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct LocalsQuery { query: tree_sitter::Query, scope: Option, definitions: Vec>, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl LocalsQuery { pub(crate) fn compile(grammar: &tree_sitter::Language, source: &str) -> Option { let query = match tree_sitter::Query::new(grammar, source) { @@ -123,7 +123,7 @@ impl LocalsQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn line_start(source: &str, line: u32) -> Option { if line == 0 { return Some(0); @@ -134,7 +134,7 @@ fn line_start(source: &str, line: u32) -> Option { .map(|(index, _)| index + 1) } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn word_node(tree: &tree_sitter::Tree, byte: usize) -> Option> { let root = tree.root_node(); let leaf = |at: usize| { @@ -145,7 +145,7 @@ fn word_node(tree: &tree_sitter::Tree, byte: usize) -> Option std::ops::RangeInclusive { - tree_sitter::MIN_COMPATIBLE_LANGUAGE_VERSION as u32..=tree_sitter::LANGUAGE_VERSION as u32 -} +pub use crate::engine::supported_abi; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PackManifest { @@ -145,6 +141,17 @@ pub enum PackError { InjectionQuery(String), } +impl From for PackError { + fn from(error: crate::engine::GrammarError) -> Self { + use crate::engine::GrammarError; + match error { + GrammarError::Abi { abi, min, max } => Self::Abi { abi, min, max }, + GrammarError::Query(error) => Self::Query(error), + GrammarError::InjectionQuery(error) => Self::InjectionQuery(error), + } + } +} + impl PackManifest { /// Checks every field the loader acts on, for packs built for /// `target`. Run before a manifest's paths or symbol are used. diff --git a/crates/diffy-syntax/src/scope.rs b/crates/diffy-syntax/src/scope.rs index b0bed63e..81abd291 100644 --- a/crates/diffy-syntax/src/scope.rs +++ b/crates/diffy-syntax/src/scope.rs @@ -8,7 +8,7 @@ use crate::{GrammarStore, LanguageId}; const CACHE_LIMIT: usize = 128; const SOURCE_LIMIT: usize = 16 << 20; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const LABEL_LIMIT: usize = 48; #[derive(Debug, Clone, PartialEq, Eq)] @@ -39,7 +39,7 @@ impl ScopeTable { .collect() } - #[cfg_attr(not(feature = "engine"), allow(dead_code))] + #[cfg_attr(not(feature = "runtime"), allow(dead_code))] fn from_spans(mut spans: Vec) -> Self { spans.sort_by(|a, b| { a.lines @@ -61,7 +61,7 @@ fn scope_table_pending( language: &LanguageId, source: &str, ) -> (ScopeTable, bool) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { if source.is_empty() || source.len() > SOURCE_LIMIT { return (ScopeTable::default(), false); @@ -84,14 +84,14 @@ fn scope_table_pending( crate::store::Tag::Unavailable => (ScopeTable::default(), false), } } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (store, language, source, SOURCE_LIMIT); (ScopeTable::default(), false) } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn scope_language(tag: &str) -> &str { match tag { "rs" => "rust", @@ -104,14 +104,14 @@ fn scope_language(tag: &str) -> &str { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] struct Pattern { query: &'static str, prefix: &'static str, suffix: &'static str, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const fn pattern(query: &'static str, prefix: &'static str, suffix: &'static str) -> Pattern { Pattern { query, @@ -120,7 +120,7 @@ const fn pattern(query: &'static str, prefix: &'static str, suffix: &'static str } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const RUST: &[Pattern] = &[ pattern( "(mod_item name: (identifier) @name body: (_)) @scope", @@ -139,7 +139,7 @@ const RUST: &[Pattern] = &[ pattern("(function_item name: (_) @name) @scope", "fn ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const ECMA: &[Pattern] = &[ pattern("(class_declaration name: (_) @name) @scope", "class ", ""), pattern( @@ -172,7 +172,7 @@ const ECMA: &[Pattern] = &[ ), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const GO: &[Pattern] = &[ pattern("(type_spec name: (_) @name) @scope", "type ", ""), pattern( @@ -183,13 +183,13 @@ const GO: &[Pattern] = &[ pattern("(function_declaration name: (_) @name) @scope", "func ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const PYTHON: &[Pattern] = &[ pattern("(class_definition name: (_) @name) @scope", "class ", ""), pattern("(function_definition name: (_) @name) @scope", "def ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const C: &[Pattern] = &[ pattern( "(namespace_definition name: (_) @name) @scope", @@ -228,7 +228,7 @@ const C: &[Pattern] = &[ ), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const ZIG: &[Pattern] = &[ pattern( "(variable_declaration (identifier) @name [(struct_declaration) (enum_declaration) (union_declaration)]) @scope", @@ -239,14 +239,14 @@ const ZIG: &[Pattern] = &[ pattern("(test_declaration (_) @name) @scope", "test ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const NIX: &[Pattern] = &[pattern( "(binding attrpath: (_) @name expression: [(function_expression) (attrset_expression) (rec_attrset_expression) (let_expression)]) @scope", "", "", )]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn patterns(language: &str) -> &'static [Pattern] { match language { "rust" => RUST, @@ -260,7 +260,7 @@ fn patterns(language: &str) -> &'static [Pattern] { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct ScopeQuery { query: tree_sitter::Query, labels: Vec<(&'static str, &'static str)>, @@ -270,7 +270,7 @@ pub(crate) struct ScopeQuery { receiver: Option, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl ScopeQuery { pub(crate) fn compile(grammar: &tree_sitter::Language, language: &str) -> Option { let usable: Vec<&Pattern> = patterns(language) @@ -334,7 +334,7 @@ impl ScopeQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn clean(text: &str) -> String { let mut out = String::new(); for word in text.split_whitespace() { @@ -456,9 +456,9 @@ impl Drop for ScopeWorker { } fn tracing_warn(error: &std::io::Error) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] tracing::warn!(%error, "scope worker unavailable"); - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = error; } diff --git a/crates/diffy-syntax/src/store.rs b/crates/diffy-syntax/src/store.rs index 4c92ef34..ce330d21 100644 --- a/crates/diffy-syntax/src/store.rs +++ b/crates/diffy-syntax/src/store.rs @@ -1,13 +1,13 @@ //! Where grammars come from: [`GrammarStore`]. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::collections::HashMap; #[cfg(feature = "engine")] use std::path::{Path, PathBuf}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::sync::{Arc, Mutex, MutexGuard}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use crate::engine::Grammar; #[cfg(feature = "engine")] use crate::pack::{self, PackManifest}; @@ -27,7 +27,7 @@ pub struct GrammarStatus { pub state: GrammarState, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Debug, Clone)] pub struct GrammarQueries { pub language: Arc, @@ -39,11 +39,11 @@ pub struct GrammarQueries { /// Finds the regions of a document written in other languages (tree-sitter /// injections), such as the scripts and styles of a Svelte component. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] pub struct Injector(Arc); -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Injector { /// The single-range regions `tree`, parsed from `source` with this /// injector's language, embeds in a named language, in match order. @@ -79,12 +79,14 @@ pub enum LanguageStatus { /// The grammars an app can highlight with, shared by every highlighter. /// Cloning is cheap and clones share state. /// -/// Packs come from, in order: +/// Grammars come from, in order: /// -/// 1. local pack roots the app opts into ([`StoreConfig::local_packs`]), -/// such as packs bundled with the app or built by `syntax-pack` in -/// development; -/// 2. with the `download` feature, the cache of a signed index the app +/// 1. with the `engine` feature, local pack roots the app opts into +/// ([`StoreConfig::local_packs`]), such as packs bundled with the app +/// or built by `syntax-pack` in development; +/// 2. with the `bundled` feature, the grammars compiled into the binary +/// ([`StoreConfig::bundled`]); +/// 3. with the `download` feature, the cache of a signed index the app /// configures ([`StoreConfig::downloads`]), downloading a missing pack /// on a background thread the first time its language is asked for. /// @@ -93,7 +95,7 @@ pub enum LanguageStatus { /// UI thread; [`crate::HighlightWorker`] makes them on its own thread. #[derive(Clone, Default)] pub struct GrammarStore { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] inner: Option>, } @@ -109,7 +111,7 @@ impl GrammarStore { Self::default() } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn new(config: StoreConfig) -> Self { Self { inner: Some(Arc::new(Inner { @@ -124,7 +126,7 @@ impl GrammarStore { } pub fn retry(&self, tag: &str) -> bool { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { let tag = tag.to_ascii_lowercase(); let mut state = lock(&inner.state); @@ -147,14 +149,14 @@ impl GrammarStore { } pub fn catalog(&self) -> Catalog { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { return inner.catalog(); } Catalog::default() } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn parser(&self, tag: &str) -> Option<(tree_sitter::Language, GrammarQueries)> { match self.grammar(&LanguageId::from_fence(tag)?) { Tag::Ready(grammar) => Some(( @@ -173,7 +175,7 @@ impl GrammarStore { /// The injection search of `tag`'s grammar, when it is loaded and has /// an injection query. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn injector(&self, tag: &str) -> Option { match self.grammar(&LanguageId::from_fence(tag)?) { Tag::Ready(grammar) if grammar.has_injections() => Some(Injector(grammar)), @@ -182,15 +184,15 @@ impl GrammarStore { } pub fn subscribe_status(&self, listener: impl Fn() -> bool + Send + 'static) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { lock(&inner.status_listeners).push(Box::new(listener)); } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = listener; } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn grammar(&self, language: &LanguageId) -> Tag { match &self.inner { Some(inner) => inner.lookup(language.as_str()), @@ -198,14 +200,14 @@ impl GrammarStore { } } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn ships_tags(&self, language: &LanguageId) -> bool { self.inner .as_ref() .is_some_and(|inner| inner.ships_tags(language.as_str())) } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn tree_sitter_language(&self, language: &LanguageId) -> Option { match self.grammar(language) { Tag::Ready(grammar) => Some(grammar.language().clone()), @@ -216,7 +218,7 @@ impl GrammarStore { /// Resolves `language`, loading its pack or starting its download on /// first use. pub fn status(&self, language: &LanguageId) -> LanguageStatus { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { return match inner.lookup(language.as_str()) { Tag::Ready(_) => LanguageStatus::Ready, @@ -232,11 +234,11 @@ impl GrammarStore { /// language resolves, whether its grammar arrived or failed. Return /// `false` to unsubscribe. Apps use it to wake their event loop. pub fn subscribe(&self, listener: impl Fn() -> bool + Send + 'static) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { lock(&inner.listeners).push(Box::new(listener)); } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = listener; } @@ -272,7 +274,7 @@ impl GrammarStore { source: &str, cancelled: &dyn Fn() -> bool, ) -> Outcome { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { self.highlight_within( language, @@ -281,7 +283,7 @@ impl GrammarStore { cancelled, ) } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (language, source, cancelled); Outcome::default() @@ -345,7 +347,7 @@ impl GrammarStore { focus: &dyn Fn() -> Option, emit: &mut dyn FnMut(Part), ) -> Outcome { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { self.highlight_windowed( language, @@ -357,7 +359,7 @@ impl GrammarStore { emit, ) } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (language, source, slice, cancelled, focus, emit); Outcome::default() @@ -365,7 +367,7 @@ impl GrammarStore { } /// [`Self::highlight_streamed`] with explicit window sizes. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] #[allow(clippy::too_many_arguments)] pub(crate) fn highlight_windowed( &self, @@ -430,7 +432,7 @@ impl GrammarStore { } /// [`Self::highlight_until`] with explicit bounds on embedded layers. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn highlight_within( &self, language: &LanguageId, @@ -482,7 +484,7 @@ pub(crate) struct Outcome { /// Bounds on embedded layers left some regions unparsed. #[cfg_attr(not(test), allow(dead_code))] pub(crate) truncated: bool, - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] #[cfg_attr(not(test), allow(dead_code))] pub(crate) work: crate::engine::Work, /// The spans went out window by window as [`Part`]s; `spans` is empty. @@ -495,7 +497,7 @@ pub(crate) struct Outcome { /// Where a streamed highlight starts its exact pass, and when it should /// stop to let other work run: asked after each window it finishes, with /// how many it finished in this call. -#[cfg_attr(not(feature = "engine"), allow(dead_code))] +#[cfg_attr(not(feature = "runtime"), allow(dead_code))] pub(crate) struct Slice<'a> { pub(crate) from: usize, /// The inexact window an earlier turn colored last. @@ -532,18 +534,22 @@ impl Outcome { /// What a [`GrammarStore`] may load. Nothing is loaded unless the app /// names it here: there is no default pack directory and no default index. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Debug, Clone, Default)] pub struct StoreConfig { + #[cfg(feature = "engine")] local_packs: Vec, + #[cfg(feature = "bundled")] + bundled: bool, /// Languages whose local packs stay pending until /// [`GrammarStore::release_deferred`], as if they were downloading. + #[cfg(feature = "engine")] deferred: Vec, #[cfg(feature = "download")] pub(crate) downloads: Option, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl StoreConfig { pub fn new() -> Self { Self::default() @@ -554,6 +560,7 @@ impl StoreConfig { /// loading one runs its code, so only name directories that only the /// app's own installer or the developer can write. Roots are searched /// in the order added, before any download. + #[cfg(feature = "engine")] pub fn local_packs(mut self, root: impl Into) -> Self { self.local_packs.push(root.into()); self @@ -563,12 +570,21 @@ impl StoreConfig { /// until [`GrammarStore::release_deferred`], the way a download leaves /// them, so tests can make a grammar arrive late without a server. #[doc(hidden)] + #[cfg(feature = "engine")] pub fn defer(mut self, languages: &[&str]) -> Self { self.deferred .extend(languages.iter().map(|l| (*l).to_owned())); self } + /// Uses the compiled-in grammars ([`crate::bundled`]), searched after + /// local packs and before downloads. + #[cfg(feature = "bundled")] + pub fn bundled(mut self) -> Self { + self.bundled = true; + self + } + /// Downloads missing packs from a signed index. #[cfg(feature = "download")] pub fn downloads(mut self, downloads: crate::download::Downloads) -> Self { @@ -577,7 +593,7 @@ impl StoreConfig { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] // Only downloads leave a language pending. #[cfg_attr(not(feature = "download"), allow(dead_code))] @@ -587,10 +603,10 @@ pub(crate) enum Tag { Unavailable, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] type Listener = Box bool + Send>; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct Inner { pub(crate) config: StoreConfig, pub(crate) state: Mutex, @@ -600,25 +616,61 @@ pub(crate) struct Inner { pub(crate) fetcher: Mutex>>, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Default)] pub(crate) struct State { /// Every tag asked for so far and what it resolved to. pub(crate) tags: HashMap, /// Tag to pack directory over the local roots, scanned on first lookup. + #[cfg(feature = "engine")] local: Option)>>, - /// Grammars by pack directory, so a language's aliases share one load. - /// `None` records a pack that failed to load. - pub(crate) loaded: HashMap>>, + /// Grammars by where they came from, so a language's aliases share one + /// load. `None` records a grammar that failed to load. + pub(crate) loaded: HashMap>>, /// [`StoreConfig::defer`]red packs have been released. + #[cfg(feature = "engine")] released: bool, /// Tags left pending by [`StoreConfig::defer`]. + #[cfg(feature = "engine")] deferred: Vec, #[cfg(feature = "download")] pub(crate) remote: crate::download::Remote, } +/// Where a grammar comes from. +#[cfg(feature = "runtime")] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum Origin { + /// A pack directory, local or in the download cache. + #[cfg(feature = "engine")] + Pack(PathBuf), + /// A grammar compiled in, by name. + #[cfg_attr(not(feature = "bundled"), allow(dead_code))] + Bundled(&'static str), +} + +/// A language the catalog lists. +#[cfg(feature = "runtime")] +struct Known { + origin: Origin, + language: String, + tags: Vec, + installed: bool, +} + #[cfg(feature = "engine")] +impl Known { + fn pack(dir: PathBuf, manifest: &PackManifest, installed: bool) -> Self { + Self { + origin: Origin::Pack(dir), + language: manifest.language.clone(), + tags: manifest.tags().map(str::to_owned).collect(), + installed, + } + } +} + +#[cfg(feature = "runtime")] pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { // A panic while holding the lock leaves maps that are still valid. mutex @@ -626,7 +678,7 @@ pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Inner { pub(crate) fn lookup(self: &Arc, tag: &str) -> Tag { let mut state = lock(&self.state); @@ -643,16 +695,26 @@ impl Inner { } fn ships_tags(&self, tag: &str) -> bool { + // Only pack sources mutate state here. + #[cfg_attr(not(feature = "engine"), allow(unused_mut))] let mut state = lock(&self.state); if let Some(Tag::Ready(grammar)) = state.tags.get(tag) { return grammar.tags().is_some(); } - let local = state + #[cfg(feature = "engine")] + if let Some((_, manifest)) = state .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)); - if let Some((_, manifest)) = local.get(tag) { + .get_or_insert_with(|| scan_local(&self.config.local_packs)) + .get(tag) + { return manifest.tags.is_some(); } + #[cfg(feature = "bundled")] + if self.config.bundled + && let Some(bundled) = crate::bundled::find(tag) + { + return bundled.tags.is_some(); + } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { return crate::download::ships_tags(downloads, &mut state, tag); @@ -661,16 +723,27 @@ impl Inner { } fn resolve(self: &Arc, state: &mut State, tag: &str) -> Tag { - let local = state + #[cfg(feature = "engine")] + if let Some((dir, manifest)) = state .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)); - if let Some((dir, manifest)) = local.get(tag).cloned() { + .get_or_insert_with(|| scan_local(&self.config.local_packs)) + .get(tag) + .cloned() + { if !state.released && self.config.deferred.contains(&manifest.language) { state.deferred.push(tag.to_owned()); return Tag::Pending; } return load(state, &dir, &manifest); } + #[cfg(feature = "bundled")] + if self.config.bundled + && let Some(bundled) = crate::bundled::find(tag) + { + return load_with(state, Origin::Bundled(bundled.name), || { + Grammar::bundled(bundled) + }); + } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { return crate::download::resolve(self, downloads, state, tag); @@ -679,63 +752,79 @@ impl Inner { } fn catalog(&self) -> Catalog { + #[cfg_attr(not(feature = "engine"), allow(unused_mut))] let mut state = lock(&self.state); - let local = state - .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)) - .clone(); - let mut packs: Vec<(PathBuf, Arc, bool)> = Vec::new(); - let mut seen = std::collections::HashSet::new(); - let mut local_packs: Vec<_> = local.values().cloned().collect(); - local_packs.sort_by(|a, b| a.0.cmp(&b.0)); - for (dir, manifest) in local_packs { - if seen.insert(manifest.language.clone()) { - packs.push((dir, manifest, true)); + let mut known: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + #[cfg(feature = "engine")] + { + let local = state + .local + .get_or_insert_with(|| scan_local(&self.config.local_packs)); + let mut local_packs: Vec<_> = local.values().cloned().collect(); + local_packs.sort_by(|a, b| a.0.cmp(&b.0)); + for (dir, manifest) in local_packs { + if seen.insert(manifest.language.clone()) { + known.push(Known::pack(dir, &manifest, true)); + } + } + } + #[cfg(feature = "bundled")] + if self.config.bundled { + for bundled in crate::bundled::GRAMMARS { + if seen.insert(bundled.name.to_owned()) { + known.push(Known { + origin: Origin::Bundled(bundled.name), + language: bundled.name.to_owned(), + tags: bundled.tags().map(str::to_owned).collect(), + installed: true, + }); + } } } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { for (dir, manifest, installed) in crate::download::packs(downloads, &mut state) { if seen.insert(manifest.language.clone()) { - packs.push((dir, manifest, installed)); + known.push(Known::pack(dir, &manifest, installed)); } } } let mut catalog = Catalog { - languages: packs.len(), + languages: known.len(), ..Catalog::default() }; - for (dir, manifest, installed) in &packs { - for tag in manifest.tags() { + for entry in &known { + for tag in &entry.tags { catalog .tags .entry(tag.to_ascii_lowercase()) - .or_insert_with(|| manifest.language.clone()); + .or_insert_with(|| entry.language.clone()); } - let pending = manifest.tags().any(|tag| { + let pending = entry.tags.iter().any(|tag| { matches!( state.tags.get(&tag.to_ascii_lowercase()), Some(Tag::Pending) ) }); #[cfg(feature = "download")] - let failure = state.remote.failure(&manifest.language); + let failure = state.remote.failure(&entry.language); #[cfg(not(feature = "download"))] let failure: Option = None; - let grammar_state = if matches!(state.loaded.get(dir), Some(Some(_))) { + let grammar_state = if matches!(state.loaded.get(&entry.origin), Some(Some(_))) { Some(GrammarState::Ready) } else if let Some(reason) = failure { Some(GrammarState::Failed(reason)) } else if pending { Some(GrammarState::Downloading) - } else if *installed { + } else if entry.installed { Some(GrammarState::Installed) } else { None }; if let Some(grammar_state) = grammar_state { catalog.statuses.push(GrammarStatus { - language: manifest.language.clone(), + language: entry.language.clone(), state: grammar_state, }); } @@ -744,6 +833,7 @@ impl Inner { catalog } + #[cfg(feature = "engine")] pub(crate) fn notify(&self) { lock(&self.listeners).retain(|listener| listener()); self.notify_status(); @@ -757,22 +847,24 @@ impl Inner { /// Loads (once per directory) the pack in `dir`. #[cfg(feature = "engine")] pub(crate) fn load(state: &mut State, dir: &Path, manifest: &PackManifest) -> Tag { - load_with(state, dir, || Grammar::load(dir, manifest)) + load_with(state, Origin::Pack(dir.to_path_buf()), || { + Grammar::load(dir, manifest) + }) } -#[cfg(feature = "engine")] -pub(crate) fn load_with( +#[cfg(feature = "runtime")] +pub(crate) fn load_with( state: &mut State, - dir: &Path, - load: impl FnOnce() -> Result, + origin: Origin, + load: impl FnOnce() -> Result, ) -> Tag { let grammar = state .loaded - .entry(dir.to_path_buf()) - .or_insert_with(|| match load() { + .entry(origin) + .or_insert_with_key(|origin| match load() { Ok(grammar) => Some(Arc::new(grammar)), Err(error) => { - tracing::warn!(pack = %dir.display(), %error, "syntax pack failed to load"); + tracing::warn!(?origin, %error, "grammar failed to load"); None } }); diff --git a/crates/diffy-syntax/src/symbols.rs b/crates/diffy-syntax/src/symbols.rs index 7b123e28..a438a673 100644 --- a/crates/diffy-syntax/src/symbols.rs +++ b/crates/diffy-syntax/src/symbols.rs @@ -1,7 +1,7 @@ -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::sync::Arc; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use crate::{GrammarStore, LanguageId}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -40,7 +40,7 @@ impl SymbolKind { .unwrap_or(Self::Other) } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] fn from_capture(name: &str) -> Option { let kind = name.strip_prefix("definition.")?; Some(match kind { @@ -68,7 +68,7 @@ pub struct Symbol { pub end_line: u32, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct TagsQuery { query: tree_sitter::Query, kinds: Vec>, @@ -76,7 +76,7 @@ pub(crate) struct TagsQuery { fingerprint: u64, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl TagsQuery { pub(crate) fn compile( grammar: &tree_sitter::Language, @@ -148,7 +148,7 @@ impl TagsQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn fnv1a(parts: &[&[u8]]) -> u64 { let mut hash: u64 = 0xcbf2_9ce4_8422_2325; for part in parts { @@ -162,14 +162,14 @@ fn fnv1a(parts: &[&[u8]]) -> u64 { hash } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] pub struct Tagger { grammar: Arc, query: Arc, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl std::fmt::Debug for Tagger { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Tagger") @@ -179,14 +179,14 @@ impl std::fmt::Debug for Tagger { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub enum TaggerLookup { Ready(Tagger), Pending, Unsupported, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Tagger { pub fn fingerprint(&self) -> u64 { self.query.fingerprint @@ -204,7 +204,7 @@ impl Tagger { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl GrammarStore { pub fn tagger(&self, language: &LanguageId) -> TaggerLookup { if !self.ships_tags(language) { diff --git a/crates/diffy-syntax/src/tests.rs b/crates/diffy-syntax/src/tests.rs index bb1e71c1..374b7852 100644 --- a/crates/diffy-syntax/src/tests.rs +++ b/crates/diffy-syntax/src/tests.rs @@ -1535,3 +1535,206 @@ mod navigate { assert_eq!(lookup.import, Some(pos(0, 26))); } } + +#[cfg(feature = "bundled")] +mod bundled { + use super::*; + use HighlightKind::*; + + /// Language, a tag that reaches it, sample (file in + /// tools/syntax-pack/samples, or inline source), and text that must get + /// each kind. + type Case = ( + &'static str, + &'static str, + &'static str, + &'static [(&'static str, HighlightKind)], + ); + const CASES: &[Case] = &[ + ( + "c", + "h", + "c.c", + &[ + ("static", Keyword), + ("greet", Function), + ("/* Prints", Comment), + ], + ), + ( + "cpp", + "cc", + "cpp.cpp", + &[("namespace", Keyword), ("\"hello, \"", String)], + ), + ( + "css", + "css", + "css.css", + &[("@media", Keyword), ("color", Property), ("4px", Number)], + ), + ( + "go", + "golang", + "go.go", + &[("func", Keyword), ("Greeter", Type), ("42", Number)], + ), + ( + "html", + "htm", + "html.html", + &[("html", Tag), ("lang", Attribute)], + ), + ( + "javascript", + "jsx", + "javascript.js", + &[("function", Keyword), ("p", Tag)], + ), + ( + "json", + "jsonc", + "json.json", + &[("\"quark\"", String), ("1", Number), ("true", Constant)], + ), + ( + "kotlin", + "kts", + "fun main() {\n val greeting = \"hi\"\n}\n", + &[("fun", Keyword), ("\"hi\"", String)], + ), + // Inline spans and the fence arrive through injections. + ( + "markdown", + "md", + "markdown.md", + &[("Hello", Keyword), ("bold", Type), ("fn", Keyword)], + ), + ( + "markdown_inline", + "markdown_inline", + "markdown_inline.md", + &[("emphasis", Attribute), ("code", String)], + ), + ( + "python", + "py", + "python.py", + &[("class", Keyword), ("greet", Function), ("42", Number)], + ), + ( + "rust", + "rs", + "rust.rs", + &[("fn", Keyword), ("Greeter", Type), ("// Says", Comment)], + ), + ( + "swift", + "swift", + "func greet(name: String) -> String {\n return \"hi\"\n}\n", + &[("func", Keyword), ("\"hi\"", String)], + ), + ( + "toml", + "toml", + "toml.toml", + &[ + ("# Package", Comment), + ("\"quark\"", String), + ("true", Constant), + ], + ), + ( + "tsx", + "tsx", + "tsx.tsx", + &[("type", Keyword), ("Props", Type), ("div", Tag)], + ), + ( + "typescript", + "mts", + "typescript.ts", + &[("interface", Keyword), ("Greeter", Type)], + ), + ( + "yaml", + "yml", + "yaml.yaml", + &[ + ("name", Property), + ("true", Constant), + ("# Workflow", Comment), + ], + ), + ]; + + fn sample(sample: &str) -> std::string::String { + if sample.contains('\n') { + return sample.to_owned(); + } + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tools/syntax-pack/samples") + .join(sample); + std::fs::read_to_string(&path).unwrap() + } + + // Needs a default-set bundle (`syntax-pack bundle`, then + // DIFFY_SYNTAX_BUNDLE). + #[test] + fn bundled_grammars_resolve_by_tag_and_highlight_their_samples() { + if crate::bundled::languages().len() == 0 { + eprintln!("skipping: built without DIFFY_SYNTAX_BUNDLE"); + return; + } + let store = GrammarStore::new(StoreConfig::new().bundled()); + + for &(name, tag, file, expected) in CASES { + assert!( + crate::bundled::languages().any(|l| l == name), + "{name} is not bundled" + ); + assert_eq!(store.status(&language(tag)), LanguageStatus::Ready, "{tag}"); + let source = sample(file); + let spans = highlight(&store, &language(tag), &source); + for &(text, kind) in expected { + let found = source.match_indices(text).any(|(at, _)| { + spans + .iter() + .any(|span| span.kind == kind && span.range().contains(&at)) + }); + assert!(found, "{name}: {text:?} is not {kind:?}"); + } + } + let rust = LanguageId::from_path("src/lib.rs").unwrap(); + assert_eq!(store.status(&rust), LanguageStatus::Ready); + } + + #[test] + fn the_catalog_lists_bundled_grammars_only_when_the_store_opts_in() { + if crate::bundled::languages().len() == 0 { + return; + } + let store = GrammarStore::new(StoreConfig::new().bundled()); + store.status(&language("rs")); + + let catalog = store.catalog(); + + assert_eq!(catalog.languages, crate::bundled::languages().len()); + assert_eq!( + catalog.tags.get("rs").map(std::string::String::as_str), + Some("rust") + ); + let state = |name: &str| { + catalog + .statuses + .iter() + .find(|status| status.language == name) + .map(|status| status.state.clone()) + }; + assert_eq!(state("rust"), Some(GrammarState::Ready)); + assert_eq!(state("go"), Some(GrammarState::Installed)); + let plain = GrammarStore::new(StoreConfig::new()); + assert_eq!(plain.status(&language("rs")), LanguageStatus::Unavailable); + assert_eq!(plain.catalog().languages, 0); + } +} diff --git a/src/app/navigation/view.rs b/src/app/navigation/view.rs index 917327ff..c5cf31f6 100644 --- a/src/app/navigation/view.rs +++ b/src/app/navigation/view.rs @@ -729,7 +729,7 @@ fn item_row(file: &ManifestFile, index: u32, tree: bool, theme: &Theme) -> AnyEl file: file.key, revision: file.revision, side: item.side, - lines: item.line..item.line + 1, + lines: item.lines.start..item.lines.start + 1, }; let mono = TextStyle::MonoSmall.font_size(theme); let small = TextStyle::BodySmall.font_size(theme); diff --git a/src/app/viewport/pipeline/state.rs b/src/app/viewport/pipeline/state.rs index c6d57110..c1d7b1fa 100644 --- a/src/app/viewport/pipeline/state.rs +++ b/src/app/viewport/pipeline/state.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::rc::Rc; use std::sync::Arc; +use diffy_compare::semantic_tag; use diffy_diff::{OutlineItem, Placeholder, SlotState, SourceCoverage, SourcePurpose}; use crate::contracts::commands::{SettingsSnapshot, ToastKind, ToastUpdate}; @@ -20,7 +21,6 @@ pub use crate::contracts::viewport::pipeline::{ WORKING_SET_LINES, }; use crate::core::compare::git::bucket_refs; -use crate::core::compare::semantic::semantic_tag; use crate::core::compare::{CompareEntry, ComparePhase, RendererKind}; use crate::core::vcs::model::{ChangeBucket, VcsCompareSpec}; use crate::platform::persistence::{PersistedCompare, Settings}; diff --git a/src/core/compare/git.rs b/src/core/compare/git.rs index 14754471..4cef2dab 100644 --- a/src/core/compare/git.rs +++ b/src/core/compare/git.rs @@ -1,9 +1,7 @@ use std::sync::Arc; -use diffy_diff::{ - DiffDocument, DiffStats, DocumentBuilder, FileMeta, FileStatus, ObjectId, SourceCoverage, - TextStore, diff_from_changes, myers::Change, -}; +use diffy_compare::{FileInput, binary_document, is_binary, line_diff}; +use diffy_diff::{DiffDocument, DiffStats, FileMeta, FileStatus, ObjectId}; use gix::bstr::{BStr, ByteSlice}; use gix::diff::blob::{Algorithm, InternedInput, diff_with_slider_heuristics}; @@ -21,7 +19,6 @@ use crate::core::vcs::model::ChangeBucket; const LOADING_FILE_EMIT_STRIDE: usize = 16; const LINE_STATS_MAX_WORKERS: usize = 8; const DEFERRED_STATS_MIN_FILES_PER_WORKER: usize = 16; -pub const CONTEXT_LINES: u32 = 3; pub fn compare( spec: &CompareSpec, @@ -256,12 +253,12 @@ fn worktree_document( }); let (old, new) = (old.unwrap_or_default(), new.unwrap_or_default()); if renderer == RendererKind::Semantic { - let changed = crate::core::compare::semantic::ChangedPath { + let changed = FileInput { status: meta.status, old_path: meta.old_path.as_deref().map(str::to_owned), new_path: meta.new_path.as_deref().map(str::to_owned), - old_content: old, - new_content: new, + old, + new, }; let mut doc = crate::core::compare::semantic::document_from_changed(&changed)?; doc.set_identity( @@ -272,7 +269,7 @@ fn worktree_document( return Ok(Some(doc)); } let _ = renderer; - document_from_bytes(meta, &old, &new).map(Some) + Ok(Some(line_diff(meta, &old, &new)?)) } pub fn compare_path( @@ -302,7 +299,7 @@ fn push_untracked(output: &mut CompareOutput, git: &GitService, only: Option<&st status: FileStatus::Added, ..FileMeta::default() }; - output.push_document(Arc::new(document_from_bytes(meta, &[], &new)?)); + output.push_document(Arc::new(line_diff(meta, &[], &new)?)); } Ok(()) } @@ -310,7 +307,7 @@ fn push_untracked(output: &mut CompareOutput, git: &GitService, only: Option<&st pub fn compare_entry(entry: &CompareEntry, git: &GitService) -> Result> { let meta = entry.file_meta(); if entry.is_binary() { - return binary_document(meta).map(Some); + return Ok(Some(binary_document(meta)?)); } let loadable = match entry.status() { FileStatus::Added => entry.new_id.is_some(), @@ -323,7 +320,7 @@ pub fn compare_entry(entry: &CompareEntry, git: &GitService) -> Result Result { @@ -344,7 +341,7 @@ pub fn compare_text(left: &str, right: &str, display_path: &str) -> Result Result { - if is_binary_bytes(old) || is_binary_bytes(new) { - return binary_document(meta); - } - let changes = line_changes(old, new); - let (old, new) = (TextStore::from_bytes(old), TextStore::from_bytes(new)); - meta.binary = false; - diff_from_changes(meta, old, new, &changes, CONTEXT_LINES) - .map_err(|error| DiffyError::Parse(error.to_string())) -} - -pub(crate) fn binary_document(mut meta: FileMeta) -> Result { - meta.binary = true; - let mut builder = DocumentBuilder::new(); - builder - .file( - meta, - SourceCoverage::PatchOnly, - TextStore::default(), - TextStore::default(), - ) - .finish() - .map_err(|error| DiffyError::Parse(error.to_string()))?; - Ok(builder.build()) -} - -pub(crate) fn line_changes(old: &[u8], new: &[u8]) -> Vec { - let input = InternedInput::new(old, new); - diff_with_slider_heuristics(Algorithm::Histogram, &input) - .hunks() - .map(|hunk| Change { - old: hunk.before, - new: hunk.after, - }) - .collect() -} - pub(crate) fn line_stats(old: &[u8], new: &[u8]) -> (u32, u32) { let input = InternedInput::new(old, new); let diff = diff_with_slider_heuristics(Algorithm::Histogram, &input); @@ -469,7 +425,7 @@ fn output_from_gix_changes( continue; }; let meta = entry_from_gix_change(change).file_meta(); - let doc = document_from_bytes(meta, old.as_bytes(), new.as_bytes())?; + let doc = line_diff(meta, old.as_bytes(), new.as_bytes())?; output.push_document(Arc::new(doc)); } Ok(output) @@ -612,7 +568,7 @@ fn entry_stats_with_trees( let (Some(old), Some(new)) = (old, new) else { return Ok(None); }; - if is_binary_bytes(old.as_bytes()) || is_binary_bytes(new.as_bytes()) { + if is_binary(old.as_bytes()) || is_binary(new.as_bytes()) { return Ok(Some(known(0, 0))); } let (additions, deletions) = line_stats(old.as_bytes(), new.as_bytes()); @@ -840,10 +796,6 @@ pub(crate) fn load_gix_blob_id( )) } -pub(crate) fn is_binary_bytes(bytes: &[u8]) -> bool { - bytes.iter().take(8192).any(|byte| *byte == 0) -} - pub(crate) fn deferred_stats_worker_count(file_count: usize) -> usize { let available = std::thread::available_parallelism() .map(usize::from) diff --git a/src/core/compare/semantic.rs b/src/core/compare/semantic.rs index 5a80931e..cd6662f4 100644 --- a/src/core/compare/semantic.rs +++ b/src/core/compare/semantic.rs @@ -1,64 +1,31 @@ use std::collections::HashMap; -use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use std::time::Duration; -use diffy_diff::{ - AlignedRow, ChangeIntensity, DiffDocument, DocumentBuilder, FileMeta, FileStatus, SemanticSpan, - Side, SourceCoverage, TextStore, -}; -use diffy_semantic::{Limits, Move, OutlineItem, Queries, Rename, Row, Rules}; -use diffy_syntax::{GrammarQueries, GrammarStore, Injector, LanguageId}; +use diffy_compare::{FileDiff, FileInput, Observer, Options}; +#[cfg(feature = "difftastic")] +use diffy_diff::{ChangeIntensity, FileMeta, SemanticSpan, TextStore}; +use diffy_diff::{DiffDocument, FileStatus}; +#[cfg(feature = "difftastic")] +use diffy_semantic::Row; #[cfg(feature = "difftastic")] use vendored_difftastic::{ ChangeIntensity as DftIntensity, ChangeSpan, DiffRequest, DiffStatus, SemanticDiffResult, }; -use crate::core::compare::git::{ - binary_document, document_from_bytes, is_binary_bytes, line_changes, resolve_compare_refs, -}; +use crate::core::compare::git::resolve_compare_refs; use crate::core::compare::output::{CompareEntry, CompareOutput}; use crate::core::compare::progress::{ComparePhase, ProgressSink}; use crate::core::compare::spec::CompareSpec; use crate::core::compare::stats::COMPARE_SUMMARY_FILE_LIMIT; -use crate::core::error::{DiffyError, Result}; +#[cfg(feature = "difftastic")] +use crate::core::error::DiffyError; +use crate::core::error::Result; use crate::core::syntax::grammar::Grammars; use crate::core::vcs::git::{ GitService, INDEX_REF, StatusItem, StatusScope, WORKDIR_REF, mentions, }; const LOADING_FILE_EMIT_STRIDE: u32 = 16; -const MAX_WORKERS: usize = 4; -const MIN_FILES_FOR_PARALLEL: usize = 4; -const WORKER_STACK_BYTES: usize = 16 << 20; -const FILE_TIME_BUDGET: Duration = Duration::from_millis(400); - -#[derive(Debug, Clone)] -pub struct ChangedPath { - pub status: FileStatus, - pub old_path: Option, - pub new_path: Option, - pub old_content: Vec, - pub new_content: Vec, -} - -impl ChangedPath { - fn display_path(&self) -> &str { - self.new_path - .as_deref() - .or(self.old_path.as_deref()) - .unwrap_or_default() - } - - fn meta(&self) -> FileMeta { - FileMeta { - old_path: self.old_path.as_deref().map(Arc::from), - new_path: self.new_path.as_deref().map(Arc::from), - status: self.status, - ..FileMeta::default() - } - } -} pub fn status_from_label(label: &str) -> FileStatus { match label.as_bytes().first() { @@ -125,12 +92,12 @@ pub fn compare_status_item(item: &StatusItem, git: &GitService) -> Result FileStatus::Deleted, _ => status_from_label(&item.status), }; - let changed = ChangedPath { + let changed = FileInput { status, old_path: old_content.as_ref().map(|_| old_path.to_owned()), new_path: new_content.as_ref().map(|_| item.path.clone()), - old_content: old_content.unwrap_or_default(), - new_content: new_content.unwrap_or_default(), + old: old_content.unwrap_or_default(), + new: new_content.unwrap_or_default(), }; let vet = rename_vet(git, old_ref.unwrap_or(new_ref), new_ref); compare_changed_paths(vec![changed], None, Some(&vet)) @@ -147,12 +114,12 @@ pub fn compare_text(left: &str, right: &str, display_path: &str) -> Result( } } +/// Per-file energy attribution and throttled `LoadingFiles` progress. +struct LoadingFiles<'a> { + reporter: Option<&'a dyn ProgressSink>, +} + +impl Observer for LoadingFiles<'_> { + fn file(&self, path: &str, work: &mut dyn FnMut()) { + let _work = crate::platform::energy::work(crate::platform::energy::Kind::Semantic) + .detail(path.to_owned()); + work(); + } + + fn progress(&self, seen: u32, total: u32) { + if let Some(r) = self.reporter + && (seen.is_multiple_of(LOADING_FILE_EMIT_STRIDE) || seen == total) + { + r.phase(ComparePhase::LoadingFiles { + files_seen: seen, + files_total: total, + }); + } + } +} + pub fn compare_changed_paths( - changed: Vec, + changed: Vec, reporter: Option<&dyn ProgressSink>, vet: Option<&diffy_semantic::Vet<'_>>, ) -> Result { - let files_total = changed.len() as u32; if let Some(r) = reporter { r.phase(ComparePhase::LoadingFiles { files_seen: 0, - files_total, + files_total: changed.len() as u32, }); } - let progress = AtomicU32::new(0); - let load = |changed: &ChangedPath| { - let doc = document_from_changed_vetted(changed, vet)?; - let seen = progress.fetch_add(1, Ordering::Relaxed).saturating_add(1); - if let Some(r) = reporter - && (seen.is_multiple_of(LOADING_FILE_EMIT_STRIDE) || seen == files_total) - { - r.phase(ComparePhase::LoadingFiles { - files_seen: seen, - files_total, - }); - } - Ok(Arc::new(doc)) - }; - let docs = load_parallel(&changed, worker_count(changed.len()), &load)?; + let store = Grammars::shared().store(); + let observer = LoadingFiles { reporter }; + let mut options = Options::new(&store); + options.vet = vet; + options.observer = Some(&observer); + let diffs = diffy_compare::diff_files(&changed, &options)?; let mut output = CompareOutput::default(); - for doc in docs { - output.push_document(doc); + for (input, diff) in changed.iter().zip(diffs) { + log_fallback(input, &diff); + output.push_document(Arc::new(diff.document)); } Ok(output) } -fn load_parallel( - changed: &[ChangedPath], - workers: usize, - load: &(dyn Fn(&ChangedPath) -> Result> + Sync), -) -> Result>> { - let next = AtomicUsize::new(0); - let mut done = std::thread::scope(|scope| { - let handles = (0..workers) - .map(|index| { - std::thread::Builder::new() - .name(format!("diffy-semantic-{index}")) - .stack_size(WORKER_STACK_BYTES) - .spawn_scoped(scope, || { - let mut done = Vec::new(); - loop { - let at = next.fetch_add(1, Ordering::Relaxed); - let Some(item) = changed.get(at) else { - return done; - }; - let _work = crate::platform::energy::work( - crate::platform::energy::Kind::Semantic, - ) - .detail(item.display_path().to_string()); - done.push((at, load(item))); - } - }) - .map_err(|error| { - DiffyError::General(format!("semantic worker setup failed: {error}")) - }) - }) - .collect::>>()?; - let mut done = Vec::with_capacity(changed.len()); - for handle in handles { - let part = handle - .join() - .map_err(|_| DiffyError::General("semantic worker panicked".to_owned()))?; - done.extend(part); - } - Ok::<_, DiffyError>(done) - })?; - done.sort_unstable_by_key(|(at, _)| *at); - done.into_iter().map(|(_, doc)| doc).collect() -} - -pub fn document_from_changed(changed: &ChangedPath) -> Result { - document_from_changed_vetted(changed, None) +/// One file's semantic diff, on the calling thread. +pub fn document_from_changed(changed: &FileInput) -> Result { + let store = Grammars::shared().store(); + let diff = diffy_compare::diff_file(changed, &Options::new(&store))?; + log_fallback(changed, &diff); + Ok(diff.document) } -fn document_from_changed_vetted( - changed: &ChangedPath, - vet: Option<&diffy_semantic::Vet<'_>>, -) -> Result { - let store = Grammars::shared().store(); - let (doc, fallback) = - document_from_native_with(changed, |tag| semantic_grammar(&store, tag), vet)?; - if let Some(reason) = fallback { +fn log_fallback(changed: &FileInput, diff: &FileDiff) { + if let Some(reason) = &diff.fallback { tracing::info!( target: "diffy::semantic", path = %changed.display_path(), - reason, - old_bytes = changed.old_content.len(), - new_bytes = changed.new_content.len(), + reason = %reason, + old_bytes = changed.old.len(), + new_bytes = changed.new.len(), "semantic diff fell back to line diff" ); } - Ok(doc) } #[cfg(feature = "difftastic")] -pub fn document_from_difftastic(changed: &ChangedPath) -> Result { - let meta = changed.meta(); - if is_binary_bytes(&changed.old_content) || is_binary_bytes(&changed.new_content) { - return binary_document(meta); - } - let old = TextStore::from_bytes(&changed.old_content); - let new = TextStore::from_bytes(&changed.new_content); - if changed.status == FileStatus::Added || changed.old_path.is_none() { - return whole_file(meta, old, new, Side::New); - } - if changed.status == FileStatus::Deleted || changed.new_path.is_none() { - return whole_file(meta, old, new, Side::Old); - } - if changed.old_content == changed.new_content { - return build(DocumentBuilder::new(), meta, old, new, |_| {}); - } +pub fn document_from_difftastic(changed: &FileInput) -> Result { + let (meta, old, new) = match diffy_compare::prepare(changed)? { + diffy_compare::Prepared::Done(doc) => return Ok(*doc), + diffy_compare::Prepared::Text { meta, old, new } => (meta, old, new), + }; let result = vendored_difftastic::diff_bytes_semantic(DiffRequest { display_path: changed.display_path(), lhs_path: changed.old_path.as_deref().map(std::path::Path::new), @@ -345,66 +269,14 @@ pub fn document_from_difftastic(changed: &ChangedPath) -> Result { }) .map_err(|error| DiffyError::General(format!("difftastic failed: {error}")))?; log_line_fallback(changed, &result); - match result.status { - DiffStatus::Binary => binary_document(meta), + let doc = match result.status { + DiffStatus::Binary => diffy_compare::binary_document(meta)?, DiffStatus::Changed if !result.chunks.is_empty() => { - document_from_semantic(meta, old, new, &result) + document_from_semantic(meta, old, new, &result)? } - _ => document_from_bytes(meta, &changed.old_content, &changed.new_content), - } -} - -fn whole_file(meta: FileMeta, old: TextStore, new: TextStore, side: Side) -> Result { - let lines = match side { - Side::Old => old.line_count(), - Side::New => new.line_count(), + _ => diffy_compare::line_diff(meta, &changed.old, &changed.new)?, }; - let (old, new) = match side { - Side::Old => (old, TextStore::default()), - Side::New => (TextStore::default(), new), - }; - build(DocumentBuilder::new(), meta, old, new, |file| { - if lines > 0 { - match side { - Side::Old => file.hunk(0, 0).change(lines, 0), - Side::New => file.hunk(0, 0).change(0, lines), - }; - } - }) -} - -fn build( - mut builder: DocumentBuilder, - meta: FileMeta, - old: TextStore, - new: TextStore, - shape: impl FnOnce(&mut diffy_diff::FileBuilder<'_>), -) -> Result { - let mut file = builder.file(meta, SourceCoverage::Full, old, new); - shape(&mut file); - file.finish() - .map_err(|error| DiffyError::Parse(error.to_string()))?; - Ok(builder.build()) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Kind { - Context, - Change, -} - -#[derive(Clone, Copy)] -struct Segment { - kind: Kind, - old: u32, - new: u32, - shown: bool, -} - -struct HunkRange { - old: std::ops::Range, - new: std::ops::Range, - segments: std::ops::Range, + Ok(doc) } #[cfg(feature = "difftastic")] @@ -417,277 +289,15 @@ fn document_from_semantic( let chunks = sanitize(result, &old, &new); let structural = result.line_fallback_reason.is_none() && line_fallback_reason(&result.language).is_none(); - document_from_rows(meta, old, new, chunks, structural, Default::default()) -} - -pub fn document_from_native( - changed: &ChangedPath, - grammars: &GrammarStore, -) -> Result<(DiffDocument, Option)> { - document_from_native_with(changed, |tag| semantic_grammar(grammars, tag), None) -} - -pub struct SemanticGrammar { - pub name: Arc, - pub language: diffy_semantic::Language, - pub rules: Arc, - pub injector: Option, -} - -pub fn semantic_tag(path: &str) -> Option { - let id = LanguageId::from_path(path)?; - match id.as_str() { - "js" | "mjs" | "cjs" => LanguageId::from_fence("typescript"), - "jsx" => LanguageId::from_fence("tsx"), - _ => Some(id), - } -} - -pub fn semantic_grammar(store: &GrammarStore, tag: &LanguageId) -> Option { - static RULES: Mutex, Arc)>> = Mutex::new(Vec::new()); - let (language, queries) = store.parser(tag.as_str())?; - let cached = |rules: &[(Arc, Arc)]| { - rules - .iter() - .find(|(highlights, _)| Arc::ptr_eq(highlights, &queries.highlights)) - .map(|(_, rules)| rules.clone()) - }; - let found = cached(&RULES.lock().unwrap_or_else(PoisonError::into_inner)); - let rules = match found { - Some(rules) => rules, - None => { - let rules = Arc::new(compile_rules(&language, &queries)); - let mut all = RULES.lock().unwrap_or_else(PoisonError::into_inner); - match cached(&all) { - Some(rules) => rules, - None => { - all.push((queries.highlights.clone(), rules.clone())); - rules - } - } - } - }; - Some(SemanticGrammar { - injector: store.injector(tag.as_str()), - name: queries.language, - language, - rules, - }) -} - -fn compile_rules(language: &diffy_semantic::Language, grammar: &GrammarQueries) -> Rules { - let queries = Queries { - highlights: &grammar.highlights, - locals: grammar.locals.as_deref(), - }; - let own = [ - diffy_semantic::semantic_query(&grammar.language), - grammar.semantic.as_deref(), - ]; - for semantic in own.into_iter().flatten() { - match Rules::new(language, semantic, &queries) { - Ok(rules) => return rules, - Err(error) => tracing::warn!( - target: "diffy::semantic", - language = %grammar.language, - %error, - "semantic query does not compile" - ), - } - } - Rules::derived(language, &queries) -} - -pub fn document_from_native_with( - changed: &ChangedPath, - grammars: impl Fn(&LanguageId) -> Option, - vet: Option<&diffy_semantic::Vet<'_>>, -) -> Result<(DiffDocument, Option)> { - let meta = changed.meta(); - if is_binary_bytes(&changed.old_content) || is_binary_bytes(&changed.new_content) { - return Ok((binary_document(meta)?, None)); - } - let old = TextStore::from_bytes(&changed.old_content); - let new = TextStore::from_bytes(&changed.new_content); - if changed.status == FileStatus::Added || changed.old_path.is_none() { - return Ok((whole_file(meta, old, new, Side::New)?, None)); - } - if changed.status == FileStatus::Deleted || changed.new_path.is_none() { - return Ok((whole_file(meta, old, new, Side::Old)?, None)); - } - if changed.old_content == changed.new_content { - return Ok((build(DocumentBuilder::new(), meta, old, new, |_| {})?, None)); - } - let line_diff = |reason: String| { - document_from_bytes(meta.clone(), &changed.old_content, &changed.new_content) - .map(|doc| (doc, Some(reason))) - }; - let Some(tag) = semantic_tag(changed.display_path()) else { - return line_diff("no grammar for this file".to_owned()); - }; - let Some(grammar) = grammars(&tag) else { - return line_diff(format!("{tag} grammar unavailable")); - }; - let limits = Limits { - max_time: FILE_TIME_BUDGET, - ..Limits::default() - }; - let embed = |tree: &diffy_semantic::Tree, source: &str| { - let Some(injector) = &grammar.injector else { - return Vec::new(); - }; - // Regions repeat a few languages many times (every `{expression}` - // of a Svelte template), so each is looked up once. - let mut seen: Vec<(LanguageId, Option)> = Vec::new(); - let mut out = Vec::new(); - for (language, range) in injector.find(tree, source) { - let at = match seen.iter().position(|(l, _)| *l == language) { - Some(at) => at, - None => { - let found = grammars(&language); - seen.push((language, found)); - seen.len() - 1 - } - }; - if let Some(embedded) = &seen[at].1 { - out.push(diffy_semantic::Embedded { - range, - language: embedded.language.clone(), - rules: embedded.rules.clone(), - name: embedded.name.clone(), - }); - } - } - out - }; - let diff = diffy_semantic::diff_with( - &grammar.language, - &grammar.rules, - old.as_str(), - new.as_str(), - &limits, - vet, - Some(&embed), - ); - match diff { - Ok(diff) => { - let chunks = clean_rows(diff.chunks, &old, &new); - let groups = diff - .groups - .into_iter() - .map(|g| diffy_diff::LineGroupSpan { - old: g.old, - new: g.new, - }) - .collect(); - let semantics = (diff.outline, diff.renames, diff.moves, groups); - let doc = document_from_rows(meta, old, new, chunks, true, semantics)?; - Ok((doc, None)) - } - Err(fallback) => line_diff(fallback.to_string()), - } -} - -fn clean_rows(chunks: Vec>, old: &TextStore, new: &TextStore) -> Vec> { - let side = |store: &TextStore, line: Option, spans: Vec| { - let line = line.filter(|&l| l < store.line_count()); - let spans = line - .and_then(|l| store.display_line(l)) - .map(|text| clean_spans(spans, text)) - .unwrap_or_default(); - (line, spans) - }; - chunks - .into_iter() - .map(|chunk| { - chunk - .into_iter() - .map(|row| { - let [old_spans, new_spans] = row.spans; - let (old_line, old_spans) = side(old, row.old, old_spans); - let (new_line, new_spans) = side(new, row.new, new_spans); - Row { - old: old_line, - new: new_line, - spans: [old_spans, new_spans], - } - }) - .collect() - }) - .collect() -} - -fn document_from_rows( - meta: FileMeta, - old: TextStore, - new: TextStore, - chunks: Vec>, - structural: bool, - (outline, renames, moves, groups): ( - Vec, - Vec, - Vec, - Vec, - ), -) -> Result { - let segments = canonical_segments(&chunks, &old, &new); - let hunks = hunk_ranges(&segments); - let mut rows: Vec> = hunks.iter().map(|_| Vec::new()).collect(); - let mut spans: [Vec<(u32, Vec)>; 2] = [Vec::new(), Vec::new()]; - let mut seen = [ - std::collections::HashSet::new(), - std::collections::HashSet::new(), - ]; - for row in chunks.iter().flatten() { - let old_hunk = row.old.and_then(|line| find_hunk(&hunks, Side::Old, line)); - let new_hunk = row.new.and_then(|line| find_hunk(&hunks, Side::New, line)); - if let Some(h) = old_hunk.or(new_hunk) { - let range = &hunks[h]; - let aligned = AlignedRow { - old: row.old.filter(|line| range.old.contains(line)), - new: row.new.filter(|line| range.new.contains(line)), - }; - if aligned.old.is_some() || aligned.new.is_some() { - rows[h].push(aligned); - } - } - for (side, line, hunk) in [ - (Side::Old, row.old, old_hunk), - (Side::New, row.new, new_hunk), - ] { - let s = side as usize; - if let (Some(line), Some(_)) = (line, hunk) - && !row.spans[s].is_empty() - && seen[s].insert(line) - { - spans[s].push((line, row.spans[s].clone())); - } - } - } - build(DocumentBuilder::new(), meta, old, new, |file| { - file.structural(structural); - file.outline(outline); - file.renames(renames); - file.moves(moves); - file.groups(groups); - for (hunk, hunk_rows) in hunks.iter().zip(rows) { - file.hunk(hunk.old.start, hunk.new.start); - for segment in &segments[hunk.segments.clone()] { - match segment.kind { - Kind::Context => file.context(segment.old), - Kind::Change => file.change(segment.old, segment.new), - }; - } - for row in hunk_rows { - file.row(row); - } - } - for side in [Side::Old, Side::New] { - for (line, line_spans) in &spans[side as usize] { - file.spans(side, *line, line_spans); - } - } - }) + let doc = diffy_compare::document_from_rows( + meta, + old, + new, + chunks, + structural, + diffy_compare::Semantics::default(), + )?; + Ok(doc) } #[cfg(feature = "difftastic")] @@ -731,34 +341,7 @@ fn sanitize_spans(spans: &[ChangeSpan], text: &str) -> Vec { intensity: intensity(span.intensity), }) .collect(); - clean_spans(spans, text) -} - -fn clean_spans(spans: Vec, text: &str) -> Vec { - let mut out: Vec = spans - .into_iter() - .filter_map(|span| { - let start = span.start as usize; - let end = (span.end as usize).min(text.len()); - (start < end && text.is_char_boundary(start) && text.is_char_boundary(end)).then_some( - SemanticSpan { - start: start as u32, - end: end as u32, - intensity: span.intensity, - }, - ) - }) - .collect(); - out.sort_by_key(|span| (span.start, span.end)); - let mut end = 0; - out.retain(|span| { - let keep = span.start >= end; - if keep { - end = span.end; - } - keep - }); - out + diffy_compare::clean_spans(spans, text) } #[cfg(feature = "difftastic")] @@ -770,177 +353,6 @@ fn intensity(intensity: DftIntensity) -> ChangeIntensity { } } -fn is_context_row(row: &Row, old: &TextStore, new: &TextStore, lo: u32, ln: u32) -> bool { - old.line(lo) == new.line(ln) - && row - .spans - .iter() - .flatten() - .all(|span| span.intensity == ChangeIntensity::UnchangedContext) -} - -fn canonical_segments(chunks: &[Vec], old: &TextStore, new: &TextStore) -> Vec { - let mut segments = Vec::new(); - let (mut o, mut n) = (0u32, 0u32); - for chunk in chunks { - let mut first = true; - for row in chunk { - let lo = row.old.filter(|&line| line >= o); - let ln = row.new.filter(|&line| line >= n); - if lo.is_none() && ln.is_none() { - continue; - } - let (to, tn) = (lo.unwrap_or(o), ln.unwrap_or(n)); - push_unmentioned(&mut segments, old, new, o..to, n..tn, !first); - first = false; - let kind = match (lo, ln) { - (Some(a), Some(b)) if is_context_row(row, old, new, a, b) => Kind::Context, - _ => Kind::Change, - }; - segments.push(Segment { - kind, - old: u32::from(lo.is_some()), - new: u32::from(ln.is_some()), - shown: true, - }); - o = to + u32::from(lo.is_some()); - n = tn + u32::from(ln.is_some()); - } - } - push_unmentioned( - &mut segments, - old, - new, - o..old.line_count(), - n..new.line_count(), - false, - ); - segments -} - -fn push_unmentioned( - segments: &mut Vec, - old: &TextStore, - new: &TextStore, - old_lines: std::ops::Range, - new_lines: std::ops::Range, - inside: bool, -) { - let (old_len, new_len) = (old_lines.len() as u32, new_lines.len() as u32); - if old_len == 0 && new_len == 0 { - return; - } - let equal = old_len == new_len - && old_lines - .clone() - .zip(new_lines.clone()) - .all(|(a, b)| old.line(a) == new.line(b)); - if equal { - segments.push(Segment { - kind: Kind::Context, - old: old_len, - new: new_len, - shown: inside, - }); - return; - } - let changes = line_changes( - lines_bytes(old, old_lines.clone()), - lines_bytes(new, new_lines.clone()), - ); - let mut pieces = Vec::with_capacity(changes.len() * 2 + 1); - let (mut o, mut n) = (0u32, 0u32); - for change in &changes { - if change.old.start > o { - pieces.push((Kind::Context, change.old.start - o, change.new.start - n)); - } - pieces.push(( - Kind::Change, - change.old.len() as u32, - change.new.len() as u32, - )); - o = change.old.end; - n = change.new.end; - } - if o < old_len { - pieces.push((Kind::Context, old_len - o, new_len - n)); - } - let last = pieces.len().saturating_sub(1); - for (index, (kind, old, new)) in pieces.into_iter().enumerate() { - let edge = index == 0 || index == last; - segments.push(Segment { - kind, - old, - new, - shown: kind == Kind::Change || !edge || inside, - }); - } -} - -fn lines_bytes(store: &TextStore, lines: std::ops::Range) -> &[u8] { - let text = store.as_str(); - let offset = |line: u32| { - store - .line_range(line) - .map_or(text.len(), |range| range.start) - }; - &text.as_bytes()[offset(lines.start)..offset(lines.end)] -} - -fn hunk_ranges(segments: &[Segment]) -> Vec { - let mut hunks = Vec::new(); - let (mut o, mut n) = (0u32, 0u32); - let mut open: Option<(u32, u32, usize, bool)> = None; - for (index, segment) in segments.iter().enumerate() { - let in_hunk = segment.shown || segment.kind == Kind::Change; - match (&mut open, in_hunk) { - (None, true) => open = Some((o, n, index, segment.kind == Kind::Change)), - (Some(state), true) => state.3 |= segment.kind == Kind::Change, - (Some(_), false) => close(&mut hunks, open.take(), o, n, index), - (None, false) => {} - } - o += segment.old; - n += segment.new; - } - close(&mut hunks, open, o, n, segments.len()); - hunks -} - -fn close( - hunks: &mut Vec, - open: Option<(u32, u32, usize, bool)>, - o: u32, - n: u32, - end: usize, -) { - if let Some((start_o, start_n, start, true)) = open { - hunks.push(HunkRange { - old: start_o..o, - new: start_n..n, - segments: start..end, - }); - } -} - -fn find_hunk(hunks: &[HunkRange], side: Side, line: u32) -> Option { - let range = |h: &HunkRange| match side { - Side::Old => h.old.clone(), - Side::New => h.new.clone(), - }; - let index = hunks.partition_point(|h| range(h).end <= line); - (index < hunks.len() && range(&hunks[index]).contains(&line)).then_some(index) -} - -fn worker_count(files: usize) -> usize { - if files < MIN_FILES_FOR_PARALLEL { - return 1; - } - let available = std::thread::available_parallelism() - .map(usize::from) - .unwrap_or(1); - files.min(available).clamp(1, MAX_WORKERS) -} - #[cfg(feature = "difftastic")] fn line_fallback_reason(language: &str) -> Option<&str> { language @@ -949,7 +361,7 @@ fn line_fallback_reason(language: &str) -> Option<&str> { } #[cfg(feature = "difftastic")] -fn log_line_fallback(changed: &ChangedPath, result: &SemanticDiffResult) { +fn log_line_fallback(changed: &FileInput, result: &SemanticDiffResult) { if let Some(reason) = result .line_fallback_reason .as_deref() @@ -960,8 +372,8 @@ fn log_line_fallback(changed: &ChangedPath, result: &SemanticDiffResult) { path = %changed.display_path(), reason, chunks = result.chunks.len(), - old_bytes = changed.old_content.len(), - new_bytes = changed.new_content.len(), + old_bytes = changed.old.len(), + new_bytes = changed.new.len(), "difftastic semantic diff fell back to line diff" ); } @@ -972,7 +384,7 @@ fn read_changed_paths( left: &str, right: &str, entries: Vec<(String, Option, Option)>, -) -> Vec { +) -> Vec { let old_paths = entries.iter().filter_map(|(_, old, _)| old.as_deref()); let mut old_contents = git.read_file_bytes_batch_at(left, old_paths).into_iter(); let new_paths = entries.iter().filter_map(|(_, _, new)| new.as_deref()); @@ -986,12 +398,12 @@ fn read_changed_paths( let new_content = new_path .as_ref() .and_then(|_| new_contents.next().flatten()); - ChangedPath { + FileInput { status: status_from_label(&status), old_path, new_path, - old_content: old_content.unwrap_or_default(), - new_content: new_content.unwrap_or_default(), + old: old_content.unwrap_or_default(), + new: new_content.unwrap_or_default(), } }) .collect() @@ -999,190 +411,13 @@ fn read_changed_paths( #[cfg(test)] mod tests { - use diffy_diff::{ - BlockKind, ChangeIntensity as Intensity, DiffDocument, FileMeta, FileStatus, SemanticSpan, - Side, TextStore, - }; - use diffy_semantic::Row; + use diffy_diff::{FileStatus, Side}; - use super::{ - ChangedPath, clean_rows, compare, compare_path, document_from_changed, document_from_rows, - }; + use super::{compare, compare_path}; use crate::core::compare::service::tests::commit_file; use crate::core::compare::spec::{CompareMode, CompareSpec, LayoutMode, RendererKind}; use crate::core::vcs::git::{GitService, WORKDIR_REF}; - fn span(start: u32, end: u32, intensity: Intensity) -> SemanticSpan { - SemanticSpan { - start, - end, - intensity, - } - } - - fn line( - old: Option, - new: Option, - old_spans: Vec, - new_spans: Vec, - ) -> Row { - Row { - old, - new, - spans: [old_spans, new_spans], - } - } - - fn rows_document(old: &str, new: &str, chunks: Vec>) -> DiffDocument { - let (old, new) = (TextStore::new(old), TextStore::new(new)); - let chunks = clean_rows(chunks, &old, &new); - document_from_rows(meta(), old, new, chunks, true, Default::default()).unwrap() - } - - fn meta() -> FileMeta { - FileMeta { - old_path: Some("src/lib.rs".into()), - new_path: Some("src/lib.rs".into()), - status: FileStatus::Modified, - ..FileMeta::default() - } - } - - fn hunks(doc: &diffy_diff::DiffDocument) -> Vec<(u32, u32, u32, u32)> { - let h = doc.hunks(); - (0..doc.hunk_count() as usize) - .map(|i| (h.old_start[i], h.old_len[i], h.new_start[i], h.new_len[i])) - .collect() - } - - #[test] - fn semantic_rows_keep_intensities_pairing_and_full_sources() { - let old = "fn a() {}\nlet x = \"foo\";\nkeep();\n"; - let new = "fn a() {}\nlet x = \"bar\";\nextra();\nkeep();\n"; - let doc = rows_document( - old, - new, - vec![vec![ - line(Some(0), Some(0), vec![], vec![]), - line( - Some(1), - Some(1), - vec![ - span(8, 9, Intensity::UnchangedContext), - span(9, 12, Intensity::NovelWord), - span(12, 13, Intensity::UnchangedContext), - ], - vec![span(9, 12, Intensity::NovelWord)], - ), - line(None, Some(2), vec![], vec![span(0, 7, Intensity::Novel)]), - line(Some(2), Some(3), vec![], vec![]), - ]], - ); - doc.verify_integrity().unwrap(); - assert_eq!(hunks(&doc), [(1, 3, 1, 4)]); - assert_eq!( - doc.blocks().kind, - [BlockKind::Context, BlockKind::Change, BlockKind::Context] - ); - assert_eq!(doc.stats(0).known(), Some((2, 1))); - assert!(doc.prefers_structural(0)); - let intensities: Vec<_> = doc - .semantic_spans(0, Side::Old, 1) - .iter() - .map(|s| s.intensity) - .collect(); - assert_eq!( - intensities, - [ - Intensity::UnchangedContext, - Intensity::NovelWord, - Intensity::UnchangedContext - ] - ); - assert_eq!( - doc.semantic_spans(0, Side::New, 2)[0].intensity, - Intensity::Novel - ); - let rows: Vec<_> = doc.aligned_rows(0).iter().map(|r| (r.old, r.new)).collect(); - assert_eq!( - rows, - [ - (Some(0), Some(0)), - (Some(1), Some(1)), - (None, Some(2)), - (Some(2), Some(3)) - ] - ); - assert_eq!(doc.text(0, Side::New).line_count(), 4); - } - - #[test] - fn lines_that_differ_between_chunks_become_changes() { - let old = "a\nb\nsame\nc\nd\n"; - let new = "A\nb\nsame2\nc\nD\n"; - let doc = rows_document( - old, - new, - vec![ - vec![line( - Some(0), - Some(0), - vec![span(0, 1, Intensity::Novel)], - vec![span(0, 1, Intensity::Novel)], - )], - vec![line( - Some(4), - Some(4), - vec![span(0, 1, Intensity::Novel)], - vec![span(0, 1, Intensity::Novel)], - )], - ], - ); - doc.verify_integrity().unwrap(); - assert_eq!(hunks(&doc), [(1, 1, 1, 1), (3, 1, 3, 1), (5, 1, 5, 1)]); - assert_eq!(doc.stats(0).known(), Some((3, 3))); - } - - #[test] - fn separate_chunks_stay_separate_hunks_and_bad_spans_are_dropped() { - let old = "a\n1\n2\n3\n4\nb\n"; - let new = "x\n1\n2\n3\n4\ny\n"; - let doc = rows_document( - old, - new, - vec![ - vec![line( - Some(0), - Some(0), - vec![span(0, 9, Intensity::Novel)], - vec![], - )], - vec![ - line(Some(5), Some(5), vec![], vec![span(3, 2, Intensity::Novel)]), - line(Some(6), Some(6), vec![], vec![]), - ], - ], - ); - assert_eq!(hunks(&doc), [(1, 1, 1, 1), (6, 1, 6, 1)]); - assert_eq!(doc.semantic_spans(0, Side::Old, 0)[0].end, 1); - assert!(doc.semantic_spans(0, Side::New, 5).is_empty()); - } - - #[test] - fn added_files_skip_the_semantic_diff() { - let doc = document_from_changed(&ChangedPath { - status: FileStatus::Added, - old_path: None, - new_path: Some("src/new.rs".to_owned()), - old_content: Vec::new(), - new_content: b"fn new() {}\nlet x = 1;\n".to_vec(), - }) - .unwrap(); - assert_eq!(hunks(&doc), [(0, 0, 1, 2)]); - assert_eq!(doc.text(0, Side::New).line(1), Some("let x = 1;")); - assert_eq!(doc.summary(0).status, FileStatus::Added); - } - fn repo_spec(left: &str, right: &str) -> CompareSpec { CompareSpec { mode: CompareMode::TwoDot, diff --git a/src/core/error.rs b/src/core/error.rs index e4421ec8..3e03cec0 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -140,4 +140,14 @@ impl DiffyError { } } +impl From for DiffyError { + fn from(error: diffy_compare::Error) -> Self { + match error { + // Build failures stay parse errors, as the app reports them. + diffy_compare::Error::Build(error) => Self::Parse(error), + error => Self::General(error.to_string()), + } + } +} + pub type Result = std::result::Result; diff --git a/src/core/vcs/jj/service.rs b/src/core/vcs/jj/service.rs index 6dd5fd7e..8d3b9aa7 100644 --- a/src/core/vcs/jj/service.rs +++ b/src/core/vcs/jj/service.rs @@ -191,7 +191,7 @@ impl JjRepository { change: &FileChange, old_rev: &str, new_rev: &str, - ) -> Result { + ) -> Result { use crate::core::vcs::compare_port::file_status; use crate::core::vcs::model::FileChangeStatus; let old_path = match change.status { @@ -212,10 +212,10 @@ impl JjRepository { .transpose() .map(Option::unwrap_or_default) }; - Ok(semantic::ChangedPath { + Ok(diffy_compare::FileInput { status: file_status(change.status), - old_content: read(old_rev, old_path)?, - new_content: read(new_rev, new_path)?, + old: read(old_rev, old_path)?, + new: read(new_rev, new_path)?, old_path: old_path.map(str::to_owned), new_path: new_path.map(str::to_owned), }) diff --git a/tools/semantic-eval/Cargo.toml b/tools/semantic-eval/Cargo.toml index 5861833f..0c92bae8 100644 --- a/tools/semantic-eval/Cargo.toml +++ b/tools/semantic-eval/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] clap = { version = "4.5", default-features = false, features = ["derive", "error-context", "help", "std", "usage"] } diffy = { path = "../..", features = ["difftastic"] } +diffy-compare = { path = "../../crates/diffy-compare" } diffy-diff = { path = "../../crates/diffy-diff" } diffy-syntax = { path = "../../crates/diffy-syntax", features = ["download"] } diffy-semantic = { path = "../../crates/diffy-semantic", features = ["eval"] } diff --git a/tools/semantic-eval/src/engine.rs b/tools/semantic-eval/src/engine.rs index 12d32236..aca7869b 100644 --- a/tools/semantic-eval/src/engine.rs +++ b/tools/semantic-eval/src/engine.rs @@ -1,11 +1,9 @@ use std::sync::OnceLock; use std::time::Instant; -use diffy::core::compare::semantic::{ - ChangedPath, SemanticGrammar, document_from_difftastic, document_from_native_with, - semantic_grammar, semantic_tag, -}; +use diffy::core::compare::semantic::document_from_difftastic; use diffy::core::syntax::grammar::GrammarConfig; +use diffy_compare::{FileInput, Options, SemanticGrammar, semantic_grammar, semantic_tag}; use diffy_diff::{DEFAULT_CONTEXT, DiffDocument, FileStatus, diff_texts}; use diffy_syntax::{GrammarStore, LanguageId, LanguageStatus}; use serde::{Deserialize, Serialize}; @@ -37,23 +35,23 @@ pub struct Run { } pub fn run(engine: Engine, path: &str, old: &[u8], new: &[u8]) -> Result { - let changed = || ChangedPath { + let changed = || FileInput { status: FileStatus::Modified, old_path: Some(path.to_owned()), new_path: Some(path.to_owned()), - old_content: old.to_vec(), - new_content: new.to_vec(), + old: old.to_vec(), + new: new.to_vec(), }; match engine { Engine::Native => { let changed = changed(); let start = Instant::now(); - let (doc, fallback) = document_from_native_with(&changed, tagged, None) + let diff = diffy_compare::diff_file_with(&changed, &Options::new(grammars()), &tagged) .map_err(|error| error.to_string())?; Ok(Run { - doc, + doc: diff.document, micros: start.elapsed().as_micros() as u64, - fallback, + fallback: diff.fallback.map(|fallback| fallback.to_string()), }) } Engine::Difftastic => { diff --git a/tools/semantic-eval/src/main.rs b/tools/semantic-eval/src/main.rs index 64aa0a33..46ce05d4 100644 --- a/tools/semantic-eval/src/main.rs +++ b/tools/semantic-eval/src/main.rs @@ -154,9 +154,10 @@ fn show(args: ShowArgs) -> Result<(), String> { if let Some(semantics) = run.doc.semantics(0) { for item in semantics.outline() { println!( - "outline {:?} {} {} {} {:?}", + "outline {:?} {}-{} {} {} {:?}", item.side, - item.line + 1, + item.lines.start + 1, + item.lines.end, item.kind, item.name, item.change diff --git a/tools/syntax-pack/APACHE-2.0.txt b/tools/syntax-pack/APACHE-2.0.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/tools/syntax-pack/APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/syntax-pack/README.md b/tools/syntax-pack/README.md index 6773a13f..33d2991b 100644 --- a/tools/syntax-pack/README.md +++ b/tools/syntax-pack/README.md @@ -20,6 +20,32 @@ subcommand. to one language and is lowercase, and that each expected injection is a pinned language's tag. +## Bundles + +`bundle` compiles grammars into static libraries for diffy-syntax's +`bundled` feature ([usage](../../crates/diffy-syntax/PACKS.md#bundled-grammars)). + +- Each language is first built and checked as a host pack from the same + sources, which adapts its queries for any target. +- Static libraries: `-fvisibility=hidden`, PIC, and `/MD` on MSVC (Rust's + default CRT). +- All libraries are linked into one throwaway shared library for the + target, so a global symbol two grammars define fails the bundle. +- C++ scanners are refused: consumers would need the C++ runtime. +- `THIRD_PARTY_NOTICES.md` copies the top-level license files of each + grammar repository (required) and query repository; + nvim-treesitter-queries-* repositories have none, so nvim-treesitter's + Apache-2.0 text is appended. +- Cross-target bundles need `CC_` and `AR_`, as `build` + does, plus the host compiler. + +```sh +# Per target, on any runner with that target's C toolchain: +cargo run -p syntax-pack -- bundle --out "$PWD/bundle" --target aarch64-apple-darwin +# Then, for the consumer: +DIFFY_SYNTAX_BUNDLE="$PWD/bundle" cargo build --release --target aarch64-apple-darwin --features diffy-syntax/bundled +``` + ## Languages - [grammars.toml](grammars.toml): the hand-pinned core languages, each with diff --git a/tools/syntax-pack/src/main.rs b/tools/syntax-pack/src/main.rs index 13ddafe4..e96aa70a 100644 --- a/tools/syntax-pack/src/main.rs +++ b/tools/syntax-pack/src/main.rs @@ -11,6 +11,12 @@ //! does. A registry language that fails is left out and recorded in //! //skipped.json; a grammars.toml language fails the build. //! Grammars marked `generate` need tree-sitter CLI 0.26.8 ($TREE_SITTER). +//! syntax-pack bundle [LANGUAGE...] [--target TRIPLE] [--out DIR] [--sources DIR] [--work DIR] [--jobs N] +//! Compile LANGUAGEs (default: BUNDLE_DEFAULT) into static libraries for +//! TRIPLE, for diffy-syntax's `bundled` feature (DIFFY_SYNTAX_BUNDLE=DIR). +//! Each is first built and checked as a host pack. Writes //: +//! lib/, queries//, bundle.tsv, THIRD_PARTY_NOTICES.md. +//! --out defaults to /target/syntax-bundle. //! syntax-pack registry --registry FILE [--extensions FILE] [--jobs N] //! Pin every grammar in nvim-treesitter's registry.json that is not in //! grammars.toml, build and check each for this host, and rewrite @@ -68,6 +74,27 @@ const HOST: &str = env!("SYNTAX_PACK_HOST"); const KEY_VAR: &str = "DIFFY_SYNTAX_SIGNING_KEY"; const TREE_SITTER_CLI: &str = "0.26.8"; const TREE_SITTER_VAR: &str = "TREE_SITTER"; +/// `bundle`'s languages when none are named; markdown injects +/// `markdown_inline`. +const BUNDLE_DEFAULT: &[&str] = &[ + "c", + "cpp", + "css", + "go", + "html", + "javascript", + "json", + "kotlin", + "markdown", + "markdown_inline", + "python", + "rust", + "swift", + "toml", + "tsx", + "typescript", + "yaml", +]; fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); @@ -80,6 +107,7 @@ fn main() -> ExitCode { } } +#[derive(Clone)] struct Options { languages: Vec, target: String, @@ -155,7 +183,7 @@ fn run(args: &[String]) -> Result<(), String> { let options = options()?; let mut skipped = BTreeMap::new(); let results = registry::par_map(&options.languages, options.jobs, |language| { - build(&lock, language, &options).and_then(|()| { + build(&lock, language, &options).and_then(|_| { if options.target == HOST { check_language(&lock, language, &options) } else { @@ -177,6 +205,16 @@ fn run(args: &[String]) -> Result<(), String> { } write_skipped(&options, &skipped) } + Some("bundle") => { + let mut options = parse_options(&args[1..])?; + if options.languages.is_empty() { + options.languages = BUNDLE_DEFAULT.iter().map(|l| (*l).to_owned()).collect(); + } + if !args.iter().any(|arg| arg == "--out") { + options.out = workspace().join("target/syntax-bundle"); + } + bundle(&lock, &options) + } Some("check") => check(&lock, &options()?), Some("index") => index(&lock, &options()?), Some("verify") => verify(&lock, &options()?), @@ -201,7 +239,7 @@ fn run(args: &[String]) -> Result<(), String> { Ok(()) } _ => Err( - "usage: syntax-pack build|check|index|verify|remote-check|registry|public-key|list \ + "usage: syntax-pack build|bundle|check|index|verify|remote-check|registry|public-key|list \ (see src/main.rs)" .to_owned(), ), @@ -419,7 +457,9 @@ fn inputs( Ok(inputs) } -fn build(lock: &Lock, language: &str, options: &Options) -> Result<(), String> { +/// Builds `language`'s pack; returns its C source directory (generated for +/// `generate` grammars). +fn build(lock: &Lock, language: &str, options: &Options) -> Result { let grammar = lock .get(language) .ok_or_else(|| format!("{language} is not pinned"))?; @@ -560,7 +600,7 @@ fn build(lock: &Lock, language: &str, options: &Options) -> Result<(), String> { out.join(&library_name).display(), manifest.library.size ); - Ok(()) + Ok(src) } fn pack_file(dir: &Path, name: &str) -> Result { @@ -615,6 +655,21 @@ fn copy_dir(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } +fn c_build(src: &Path, target: &str, objects: &Path) -> cc::Build { + let mut build = cc::Build::new(); + build + .cargo_metadata(false) + .cargo_warnings(false) + .warnings(false) + .target(target) + .host(HOST) + .opt_level(2) + .debug(false) + .out_dir(objects) + .include(src); + build +} + /// Compiles parser.c and its scanner in `src` into the shared library /// `output` for `target`. fn compile( @@ -626,18 +681,8 @@ fn compile( ) -> Result<(), String> { let _ = fs::remove_dir_all(objects); fs::create_dir_all(objects).map_err(|e| e.to_string())?; - let mut build = cc::Build::new(); - build - .cargo_metadata(false) - .cargo_warnings(false) - .warnings(false) - .target(target) - .host(HOST) - .opt_level(2) - .debug(false) - .out_dir(objects) - .include(src) - .file(src.join("parser.c")); + let mut build = c_build(src, target, objects); + build.file(src.join("parser.c")); if scanner == Scanner::C { build.file(src.join("scanner.c")); } @@ -657,32 +702,41 @@ fn compile( .try_compile_intermediates() .map_err(|e| e.to_string())?; if scanner == Scanner::Cc { - let mut cxx = cc::Build::new(); - cxx.cargo_metadata(false) - .cargo_warnings(false) - .warnings(false) - .cpp(true) - .target(target) - .host(HOST) - .opt_level(2) - .debug(false) - .out_dir(objects) - .include(src) - .file(src.join("scanner.cc")); + let mut cxx = c_build(src, target, objects); + cxx.cpp(true).file(src.join("scanner.cc")); if !compiler.is_like_msvc() { cxx.flag("-fvisibility=hidden"); } objs.extend(cxx.try_compile_intermediates().map_err(|e| e.to_string())?); } - let mut link = if compiler.is_like_msvc() { + let mut link = shared_link(&compiler, target, objects, output)?; + link.args(&objs); + if scanner == Scanner::Cc && !compiler.is_like_msvc() { + link.arg(if target.contains("apple") { + "-lc++" + } else { + "-lstdc++" + }); + } + run_link(link, output) +} + +/// Linker command for the shared library `output`; the caller adds inputs. +fn shared_link( + compiler: &cc::Tool, + target: &str, + objects: &Path, + output: &Path, +) -> Result { + if compiler.is_like_msvc() { let mut link = cc::windows_registry::find_tool(target, "link.exe") .ok_or("link.exe not found; run from a Visual Studio developer shell")? .to_command(); link.args(["/NOLOGO", "/DLL"]) .arg(format!("/OUT:{}", output.display())) .arg(format!("/IMPLIB:{}", objects.join("parser.lib").display())); - link + Ok(link) } else { let mut link = compiler.to_command(); if target.contains("apple") { @@ -691,16 +745,11 @@ fn compile( link.args(["-shared", "-s"]); } link.arg("-o").arg(output); - link - }; - link.args(&objs); - if scanner == Scanner::Cc && !compiler.is_like_msvc() { - link.arg(if target.contains("apple") { - "-lc++" - } else { - "-lstdc++" - }); + Ok(link) } +} + +fn run_link(mut link: Command, output: &Path) -> Result<(), String> { let status = link.status().map_err(|e| format!("linker: {e}"))?; if !status.success() { return Err(format!("linking {} failed: {status}", output.display())); @@ -708,6 +757,343 @@ fn compile( Ok(()) } +// ---- Bundle ------------------------------------------------------------- + +/// The static library name of a bundled language. +fn bundle_library(language: &str) -> String { + let name: String = language + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + format!("diffy_grammar_{name}") +} + +/// File name of static library `name` on `target`, as `cc` writes and +/// rustc expects. +fn static_library_file(target: &str, name: &str) -> String { + if target.contains("msvc") { + format!("{name}.lib") + } else { + format!("lib{name}.a") + } +} + +fn bundle(lock: &Lock, options: &Options) -> Result<(), String> { + let target = &options.target; + let mut libraries = BTreeMap::new(); + for language in &options.languages { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + // Linking one would need the C++ runtime in every consumer. + if grammar.scanner == Scanner::Cc { + return Err(format!("{language}: C++ scanners cannot be bundled")); + } + if let Some(other) = libraries.insert(bundle_library(language), language) { + return Err(format!("{other} and {language} share a library name")); + } + } + let out = options.out.join(target); + let _ = fs::remove_dir_all(&out); + let lib_dir = out.join("lib"); + fs::create_dir_all(&lib_dir).map_err(|e| format!("{}: {e}", lib_dir.display()))?; + // Host packs, whatever the target: building one adapts its queries, and + // checking it tests them against the sources the bundle compiles. + let host = Options { + target: HOST.to_owned(), + out: options.work.join("bundle-packs"), + ..options.clone() + }; + let results = registry::par_map(&options.languages, options.jobs, |language| { + let src = build(lock, language, &host)?; + check_language(lock, language, &host)?; + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + let objects = options + .work + .join("bundle-objects") + .join(target) + .join(language); + let library = bundle_library(language); + eprintln!("compiling {language} into {library} for {target}"); + compile_static(&src, grammar.scanner, target, &objects, &library, &lib_dir) + .map_err(|e| format!("{language}: {e}")) + }); + for result in results { + result?; + } + + let mut table = String::from( + "# diffy-syntax bundle 1\n\ + # language\tsymbol\tabi\tlibrary\taliases\textensions\tsource sha256\n", + ); + let mut symbols = BTreeSet::new(); + for language in &options.languages { + let dir = host.out.join(HOST).join(language); + let manifest = pack::read_manifest(&dir).map_err(|e| format!("{language}: {e}"))?; + if !symbols.insert(manifest.symbol.clone()) { + return Err(format!("{language}: symbol {} is taken", manifest.symbol)); + } + let queries = out.join("queries").join(language); + fs::create_dir_all(&queries).map_err(|e| e.to_string())?; + for (name, file) in [ + ("highlights", Some(&manifest.highlights)), + ("injections", manifest.injections.as_ref()), + ("locals", manifest.locals.as_ref()), + ("semantic", manifest.semantic.as_ref()), + ("tags", manifest.tags.as_ref()), + ] { + if let Some(file) = file { + let to = queries.join(format!("{name}.scm")); + fs::copy(dir.join(&file.path), &to) + .map_err(|e| format!("{}: {e}", to.display()))?; + } + } + let list = |tags: &[String]| match tags.join(",") { + joined if joined.is_empty() => "-".to_owned(), + joined => joined, + }; + let source = manifest + .source + .as_ref() + .ok_or_else(|| format!("{language}: the pack names no source"))?; + table.push_str(&format!( + "{language}\t{}\t{}\t{}\t{}\t{}\t{}\n", + manifest.symbol, + manifest.abi, + bundle_library(language), + list(&manifest.aliases), + list(&manifest.extensions), + source.sha256, + )); + } + probe_link(options, &symbols)?; + let notices = out.join("THIRD_PARTY_NOTICES.md"); + fs::write(¬ices, third_party_notices(lock, options)?) + .map_err(|e| format!("{}: {e}", notices.display()))?; + // Written last: diffy-syntax refuses a bundle without it. + fs::write(out.join("bundle.tsv"), table).map_err(|e| e.to_string())?; + for language in &options.languages { + let path = lib_dir.join(static_library_file(target, &bundle_library(language))); + let size = fs::metadata(&path).map_err(|e| e.to_string())?.len(); + println!("{} ({size} bytes)", path.display()); + } + println!("{}", out.display()); + Ok(()) +} + +/// Compiles `src`'s parser.c and C scanner into the static `library` in +/// `lib_dir`. +fn compile_static( + src: &Path, + scanner: Scanner, + target: &str, + objects: &Path, + library: &str, + lib_dir: &Path, +) -> Result<(), String> { + let _ = fs::remove_dir_all(objects); + fs::create_dir_all(objects).map_err(|e| format!("{}: {e}", objects.display()))?; + let mut build = c_build(src, target, objects); + // PIC so shared libraries (Node addons) can link it; dynamic CRT to + // match Rust on MSVC. + build.pic(true).static_crt(false).file(src.join("parser.c")); + if scanner == Scanner::C { + build.file(src.join("scanner.c")); + } + let compiler = build.try_get_compiler().map_err(|e| e.to_string())?; + if !compiler.is_like_msvc() { + build.flag("-std=c11").flag("-fvisibility=hidden"); + } + build.try_compile(library).map_err(|e| e.to_string())?; + let file = static_library_file(target, library); + fs::copy(objects.join(&file), lib_dir.join(&file)).map_err(|e| format!("{file}: {e}"))?; + Ok(()) +} + +/// Links the whole bundle into one shared library, so clashing global +/// symbols fail here rather than in a consumer's build. +fn probe_link(options: &Options, symbols: &BTreeSet) -> Result<(), String> { + let target = &options.target; + let objects = options + .work + .join("bundle-objects") + .join(target) + .join("probe"); + let _ = fs::remove_dir_all(&objects); + fs::create_dir_all(&objects).map_err(|e| e.to_string())?; + let mut probe = String::new(); + for symbol in symbols { + probe.push_str(&format!("const void *{symbol}(void);\n")); + } + probe.push_str("const void *(*const diffy_bundle_probe[])(void) = {\n"); + for symbol in symbols { + probe.push_str(&format!(" {symbol},\n")); + } + probe.push_str("};\n"); + let source = objects.join("probe.c"); + fs::write(&source, probe).map_err(|e| e.to_string())?; + let mut build = c_build(&objects, target, &objects); + build.pic(true).static_crt(false).file(&source); + let compiler = build.try_get_compiler().map_err(|e| e.to_string())?; + let objs = build + .try_compile_intermediates() + .map_err(|e| format!("probe: {e}"))?; + let output = objects.join(match pack::library_suffix(target) { + ".dll" => "probe.dll".to_owned(), + suffix => format!("libprobe{suffix}"), + }); + let mut link = shared_link(&compiler, target, &objects, &output)?; + link.args(&objs); + let lib_dir = options.out.join(target).join("lib"); + for language in &options.languages { + link.arg(lib_dir.join(static_library_file(target, &bundle_library(language)))); + } + run_link(link, &output).map_err(|e| format!("the bundle does not link as one: {e}")) +} + +/// Per grammar: license, source repositories and commits, and their +/// license files. +fn third_party_notices(lock: &Lock, options: &Options) -> Result { + let mut text = String::from( + "# Third-party notices\n\n\ + These tree-sitter grammars and queries are compiled into this software.\n", + ); + let mut nvim_queries = false; + for language in &options.languages { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + text.push_str(&format!( + "\n## {language}\n\nLicense: {}\n\n", + grammar.license.as_deref().unwrap_or("unknown") + )); + let sources = source_repositories(lock, language, options)?; + for (index, (repo, rev, root)) in sources.into_iter().enumerate() { + text.push_str(&format!("Source: {repo} at {rev}\n\n")); + let licenses = license_files(&root)?; + // The grammar repository's C code is compiled in, so it must + // ship a license. + if licenses.is_empty() && index == 0 { + return Err(format!( + "{language}: {repo} at {rev} has no license file to ship" + )); + } + if licenses.is_empty() && repo.contains("nvim-treesitter-queries-") { + // Split from nvim-treesitter, whose queries are Apache-2.0. + nvim_queries = true; + text.push_str( + "Queries from nvim-treesitter, licensed Apache-2.0 \ + (text at the end of this file).\n\n", + ); + } else if licenses.is_empty() { + text.push_str("This repository has no license file.\n\n"); + } + for path in licenses { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let license = read_text(&path)?; + text.push_str(&format!( + "{name}:\n\n```text\n{}\n```\n\n", + license.trim_end() + )); + } + } + } + if nvim_queries { + text.push_str(&format!( + "\n## nvim-treesitter queries\n\n\ + Source: https://github.com/nvim-treesitter/nvim-treesitter\n\n\ + ```text\n{}\n```\n", + APACHE_2_0.trim_end() + )); + } + Ok(text) +} + +const APACHE_2_0: &str = include_str!("../APACHE-2.0.txt"); + +/// Repository, commit, and checkout of every source `language`'s build +/// reads: its grammar, its queries, and other entries it names +/// (`@/`, `@:q/`). +fn source_repositories( + lock: &Lock, + language: &str, + options: &Options, +) -> Result, String> { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + let mut found = vec![( + grammar.repo.clone(), + grammar.rev.clone(), + checkout(lock, language, options)?, + )]; + let names = grammar + .include + .iter() + .chain(&grammar.highlights) + .chain(&grammar.injections) + .chain(&grammar.locals) + .chain(&grammar.tags); + for name in names { + let (owner, queries) = if name.starts_with("q:") { + (language, true) + } else if let Some((other, _)) = name.strip_prefix('@').and_then(|r| r.split_once('/')) { + match other.strip_suffix(":q") { + Some(other) => (other, true), + None => (other, false), + } + } else { + continue; + }; + let (source, root) = if queries { + let source = lock + .queries_of(owner) + .ok_or_else(|| format!("{owner} has no pinned queries"))?; + (source, queries_checkout(lock, owner, options)?) + } else { + let other = lock + .get(owner) + .ok_or_else(|| format!("{owner} is not pinned"))?; + let source = Source { + repo: other.repo.clone(), + rev: other.rev.clone(), + }; + (source, checkout(lock, owner, options)?) + }; + if !found + .iter() + .any(|(repo, rev, _)| *repo == source.repo && *rev == source.rev) + { + found.push((source.repo, source.rev, root)); + } + } + Ok(found) +} + +/// License files at a checkout's top level, sorted. +fn license_files(root: &Path) -> Result, String> { + let entries = fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))?; + let mut found: Vec = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + let name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_ascii_uppercase(); + path.is_file() + && ["LICENSE", "LICENCE", "COPYING"] + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .collect(); + found.sort(); + Ok(found) +} + // ---- Check -------------------------------------------------------------- fn check(lock: &Lock, options: &Options) -> Result<(), String> {