From b0688ad5d716d70d3f42e2bea5c059a755652bdb Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:00:03 +0000 Subject: [PATCH 01/16] feat(compare): add diffy-compare contract and relicense the compare chain The headless crates the compare chain is built from (diffy-diff, diffy-semantic, diffy-syntax, diffy-compare) are MIT OR Apache-2.0; the app stays GPL-3.0. --- Cargo.lock | 9 ++ Cargo.toml | 1 + README.md | 5 +- crates/diffy-compare/Cargo.toml | 20 +++ crates/diffy-compare/LICENSE-APACHE | 202 +++++++++++++++++++++++++++ crates/diffy-compare/LICENSE-MIT | 21 +++ crates/diffy-compare/src/lib.rs | 191 +++++++++++++++++++++++++ crates/diffy-compare/src/wire.rs | 1 + crates/diffy-diff/Cargo.toml | 1 + crates/diffy-diff/LICENSE-APACHE | 202 +++++++++++++++++++++++++++ crates/diffy-diff/LICENSE-MIT | 21 +++ crates/diffy-semantic/Cargo.toml | 1 + crates/diffy-semantic/LICENSE-APACHE | 202 +++++++++++++++++++++++++++ crates/diffy-semantic/LICENSE-MIT | 21 +++ crates/diffy-syntax/Cargo.toml | 1 + crates/diffy-syntax/LICENSE-APACHE | 202 +++++++++++++++++++++++++++ crates/diffy-syntax/LICENSE-MIT | 21 +++ 17 files changed, 1121 insertions(+), 1 deletion(-) create mode 100644 crates/diffy-compare/Cargo.toml create mode 100644 crates/diffy-compare/LICENSE-APACHE create mode 100644 crates/diffy-compare/LICENSE-MIT create mode 100644 crates/diffy-compare/src/lib.rs create mode 100644 crates/diffy-compare/src/wire.rs create mode 100644 crates/diffy-diff/LICENSE-APACHE create mode 100644 crates/diffy-diff/LICENSE-MIT create mode 100644 crates/diffy-semantic/LICENSE-APACHE create mode 100644 crates/diffy-semantic/LICENSE-MIT create mode 100644 crates/diffy-syntax/LICENSE-APACHE create mode 100644 crates/diffy-syntax/LICENSE-MIT diff --git a/Cargo.lock b/Cargo.lock index ae88fec9..933343be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1243,6 +1243,15 @@ dependencies = [ "winit", ] +[[package]] +name = "diffy-compare" +version = "0.3.7" +dependencies = [ + "diffy-diff", + "diffy-semantic", + "diffy-syntax", +] + [[package]] name = "diffy-diff" version = "0.3.7" diff --git a/Cargo.toml b/Cargo.toml index 6baa9b03..a2450aca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ ".", + "crates/diffy-compare", "crates/diffy-diff", "crates/diffy-semantic", "crates/diffy-syntax", diff --git a/README.md b/README.md index 5359b916..652aaa42 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,7 @@ dx serve --hot-patch --features hot-reload ## License -GPL-3.0 +The Diffy app is GPL-3.0. The headless crates the compare chain is built +from (`diffy-diff`, `diffy-semantic`, `diffy-syntax`, `diffy-compare`) and +the Node binding are MIT OR Apache-2.0; see the `LICENSE-MIT` and +`LICENSE-APACHE` files in each. diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml new file mode 100644 index 00000000..8c6f51fb --- /dev/null +++ b/crates/diffy-compare/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "diffy-compare" +description = "Headless file comparison for Diffy: semantic diff where a grammar exists, line diff otherwise." +version.workspace = true +edition = "2024" +rust-version = "1.92" +license = "MIT OR Apache-2.0" +publish = false + +# No UI, Git, or network dependencies: callers read the files and hand over +# their bytes. Grammars come from a diffy-syntax store the caller builds. +[dependencies] +diffy-diff = { path = "../diffy-diff" } +diffy-semantic = { path = "../diffy-semantic" } +diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } + +[features] +default = [] +# The versioned, serializable display format in `wire`. +serde = [] diff --git a/crates/diffy-compare/LICENSE-APACHE b/crates/diffy-compare/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/crates/diffy-compare/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-compare/LICENSE-MIT b/crates/diffy-compare/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-compare/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-compare/src/lib.rs b/crates/diffy-compare/src/lib.rs new file mode 100644 index 00000000..db14cdb4 --- /dev/null +++ b/crates/diffy-compare/src/lib.rs @@ -0,0 +1,191 @@ +//! Headless file comparison: the chain from a changed file's bytes to a +//! [`DiffDocument`], with no UI, Git, or network dependencies. +//! +//! For each file the caller hands over its status, old and new paths, and +//! old and new bytes ([`FileInput`]). [`diff_files`] picks a language from +//! the path, loads its grammar from the caller's [`GrammarStore`], and runs +//! the semantic diff, falling back to a line diff (and saying why, see +//! [`Fallback`]) when no grammar exists or the semantic diff gives up. +//! Files are diffed on a few worker threads. +//! +//! Budgets: each file's semantic diff gets [`Options::file_budget`]; a +//! request may also set a [`Options::deadline`], after which remaining +//! files get line diffs only. [`Options::cancel`] stops the request +//! between files with [`Error::Cancelled`]. +//! +//! With the `serde` feature, [`wire`] projects results into a versioned +//! display format that non-Rust clients render from. + +// CONTRACT SKELETON: the types and signatures below are the agreed API. +// Bodies marked `todo!` are filled in by the compare-core change. + +use std::sync::atomic::AtomicBool; +use std::time::{Duration, Instant}; + +pub use diffy_diff::{DiffDocument, FileStatus}; +pub use diffy_syntax::{GrammarStore, LanguageId}; + +#[cfg(feature = "serde")] +pub mod wire; + +/// The semantic diff's default time budget per file. +pub const FILE_BUDGET: Duration = Duration::from_millis(400); + +/// One changed file, as the caller read it. +#[derive(Debug, Clone, Default)] +pub struct FileInput { + pub status: FileStatus, + /// `None` for an added file. + pub old_path: Option, + /// `None` for a deleted file. + pub new_path: Option, + pub old: Vec, + pub new: Vec, +} + +impl FileInput { + /// The new path, else the old one. + pub fn display_path(&self) -> &str { + self.new_path + .as_deref() + .or(self.old_path.as_deref()) + .unwrap_or_default() + } +} + +/// Why a file got a line diff instead of a semantic one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Fallback { + /// No language is known for the path. + NoGrammar, + /// The path names a language (this tag) whose grammar the store + /// cannot load. + GrammarUnavailable(String), + /// The semantic diff gave up: size, parse errors, work or time budget. + Semantic(diffy_semantic::Fallback), + /// The request's deadline passed before this file was diffed. + RequestBudget, +} + +impl std::fmt::Display for Fallback { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoGrammar => f.write_str("no grammar for this file"), + Self::GrammarUnavailable(tag) => write!(f, "{tag} grammar unavailable"), + Self::Semantic(fallback) => fallback.fmt(f), + Self::RequestBudget => f.write_str("request time budget exhausted"), + } + } +} + +/// One file's result. Binary files, added and deleted files, and unchanged +/// files never run the semantic diff and have no fallback. +#[derive(Debug, Clone)] +pub struct FileDiff { + /// Holds exactly one file. + pub document: DiffDocument, + pub fallback: Option, +} + +/// Hooks for callers that track progress or account for work per file. +/// Called from worker threads. +pub trait Observer: Sync { + /// Runs `work`, which diffs the file at `path`; wrap it to time or + /// attribute the work. The default just runs it. + fn file(&self, path: &str, work: &mut dyn FnMut()) { + let _ = path; + work(); + } + + /// `seen` of `total` files are done. + fn progress(&self, seen: u32, total: u32) { + let _ = (seen, total); + } +} + +pub struct Options<'a> { + /// Grammars for the semantic diff; [`GrammarStore::none`] gives line + /// diffs only. + pub grammars: &'a GrammarStore, + /// Time budget for one file's semantic diff. + pub file_budget: Duration, + /// When the whole request must be done. Files started after it get + /// line diffs ([`Fallback::RequestBudget`]); a file's semantic budget + /// never runs past it. + pub deadline: Option, + /// Set to stop the request; checked between files. + pub cancel: Option<&'a AtomicBool>, + /// Confirms candidate renames of identifiers, see + /// [`diffy_semantic::Vet`]. + pub vet: Option<&'a diffy_semantic::Vet<'a>>, + /// Worker threads; 0 picks from the file count and the machine. + pub workers: usize, + pub observer: Option<&'a dyn Observer>, +} + +impl<'a> Options<'a> { + pub fn new(grammars: &'a GrammarStore) -> Self { + Self { + grammars, + file_budget: FILE_BUDGET, + deadline: None, + cancel: None, + vet: None, + workers: 0, + observer: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + /// [`Options::cancel`] was set. + Cancelled, + /// A document failed to build; a bug, never expected. + Build(String), + /// A worker thread could not start or panicked. + Worker(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Cancelled => f.write_str("compare cancelled"), + Self::Build(error) => write!(f, "diff document failed to build: {error}"), + Self::Worker(error) => write!(f, "compare worker failed: {error}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Diffs one file on the calling thread. +pub fn diff_file(input: &FileInput, options: &Options<'_>) -> Result { + let _ = (input, options); + todo!("compare core") +} + +/// Diffs every file, in parallel when there are several, and returns the +/// results in input order. +pub fn diff_files(inputs: &[FileInput], options: &Options<'_>) -> Result> { + let _ = (inputs, options); + todo!("compare core") +} + +/// The language tag the semantic diff uses for `path`: JavaScript is +/// parsed with the TypeScript grammar, JSX with TSX. +pub fn semantic_tag(path: &str) -> Option { + let id = LanguageId::from_path(path)?; + match id.as_str() { + "js" | "mjs" | "cjs" => LanguageId::from_fence("typescript"), + "jsx" => LanguageId::from_fence("tsx"), + _ => Some(id), + } +} + +/// Whether bytes look binary: a NUL in the first 8 KiB, as Git decides. +pub fn is_binary(bytes: &[u8]) -> bool { + bytes.iter().take(8192).any(|byte| *byte == 0) +} diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs new file mode 100644 index 00000000..249eaed6 --- /dev/null +++ b/crates/diffy-compare/src/wire.rs @@ -0,0 +1 @@ +//! The versioned display format; filled in by the wire-format change. diff --git a/crates/diffy-diff/Cargo.toml b/crates/diffy-diff/Cargo.toml index 687c8a5a..9982dc96 100644 --- a/crates/diffy-diff/Cargo.toml +++ b/crates/diffy-diff/Cargo.toml @@ -4,6 +4,7 @@ description = "Diff model for Diffy: unified patch parsing, line and word diffs, version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false # No UI, text shaping, or syntax dependencies: the diff view in diffy-ui diff --git a/crates/diffy-diff/LICENSE-APACHE b/crates/diffy-diff/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/crates/diffy-diff/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-diff/LICENSE-MIT b/crates/diffy-diff/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-diff/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-semantic/Cargo.toml b/crates/diffy-semantic/Cargo.toml index 31718865..22ceb26d 100644 --- a/crates/diffy-semantic/Cargo.toml +++ b/crates/diffy-semantic/Cargo.toml @@ -4,6 +4,7 @@ description = "Structural diff of two sources parsed with tree-sitter, for Diffy version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false [dependencies] diff --git a/crates/diffy-semantic/LICENSE-APACHE b/crates/diffy-semantic/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/crates/diffy-semantic/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-semantic/LICENSE-MIT b/crates/diffy-semantic/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-semantic/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/diffy-syntax/Cargo.toml b/crates/diffy-syntax/Cargo.toml index 7c480ecc..df1aa589 100644 --- a/crates/diffy-syntax/Cargo.toml +++ b/crates/diffy-syntax/Cargo.toml @@ -4,6 +4,7 @@ description = "Tree-sitter syntax highlighting for Diffy, with grammars loaded a version.workspace = true edition = "2024" rust-version = "1.92" +license = "MIT OR Apache-2.0" publish = false # No grammar is compiled in. Grammars arrive as packs (a shared library diff --git a/crates/diffy-syntax/LICENSE-APACHE b/crates/diffy-syntax/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/crates/diffy-syntax/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/diffy-syntax/LICENSE-MIT b/crates/diffy-syntax/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/crates/diffy-syntax/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From e02a570f7ff0b0d3858044b4a29b136ca1397f06 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:16:06 +0000 Subject: [PATCH 02/16] feat(compare): add the versioned wire format Project compare results into a layout-neutral, serde-serializable display format for JavaScript clients: hunks as context/removed/added/modified rows with UTF-16 change and syntax-class spans, collapsed gaps with an expand entry point, hunk headers from the semantic outline or the grammar's scope table, outline, moves, renames, and stable fallback kinds. FORMAT.md is the client-facing spec, checked against the types by a test. --- Cargo.lock | 2 + crates/diffy-compare/Cargo.toml | 8 +- crates/diffy-compare/FORMAT.md | 380 ++++++++++ crates/diffy-compare/src/wire.rs | 997 ++++++++++++++++++++++++- crates/diffy-compare/src/wire/tests.rs | 749 +++++++++++++++++++ 5 files changed, 2134 insertions(+), 2 deletions(-) create mode 100644 crates/diffy-compare/FORMAT.md create mode 100644 crates/diffy-compare/src/wire/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 933343be..cdfaf1c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,8 @@ dependencies = [ "diffy-diff", "diffy-semantic", "diffy-syntax", + "serde", + "serde_json", ] [[package]] diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml index 8c6f51fb..c667b8cf 100644 --- a/crates/diffy-compare/Cargo.toml +++ b/crates/diffy-compare/Cargo.toml @@ -13,8 +13,14 @@ publish = false diffy-diff = { path = "../diffy-diff" } diffy-semantic = { path = "../diffy-semantic" } diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } +# Only for `wire`; diffy-syntax's engine already builds serde, so the +# feature adds no new crate. +serde = { version = "1.0", default-features = false, features = ["derive", "std"], optional = true } [features] default = [] # The versioned, serializable display format in `wire`. -serde = [] +serde = ["dep:serde"] + +[dev-dependencies] +serde_json = "1.0" diff --git a/crates/diffy-compare/FORMAT.md b/crates/diffy-compare/FORMAT.md new file mode 100644 index 00000000..fa6f4e70 --- /dev/null +++ b/crates/diffy-compare/FORMAT.md @@ -0,0 +1,380 @@ +# Diffy display format, version 1 + +The JSON that Diffy's compare produces for clients to render diffs from +(`diffy_compare::wire`, behind the crate's `serde` feature). It is +layout-neutral: one row list renders both split and unified views, and +nothing needs recomputing on the client. + +## Versioning + +Every document carries `version`. It changes only for a breaking change: +removing or renaming a field or enum value, changing a field's type, +meaning, or coordinates, or making an optional field required. Adding an +optional field is not breaking and keeps the version, so clients must +ignore fields they do not know. The one enum that may gain values without +a bump is the syntax `class`; treat unknown classes as plain text. + +## Conventions + +- Field names are `snake_case`; enum values are `snake_case` strings. +- Optional fields are omitted when empty: an absent string or object means + none, an absent array means `[]`, an absent boolean means `false`. +- Line numbers are 1-based. +- Ranges are half-open: `start` is included, `end` is not. +- **Span offsets are UTF-16 code units within the row's `text`**, the way + JavaScript indexes strings: `text.slice(span.start, span.end)` is the + span's text. +- `text` is one line as Diffy displays it: no line terminator, no trailing + `\r` of a CRLF line, and each byte that is not valid UTF-8 shown as + U+FFFD (`�`). Whether the last line ends with a newline is stated on the + file (`old_missing_newline`, `new_missing_newline`). + +## Document + +| Field | Type | | +| --- | --- | --- | +| `version` | number | The format version, `1`. | +| `files` | File[] | In the order the files were requested. | + +## File + +| Field | Type | | +| --- | --- | --- | +| `status` | `"modified"` \| `"added"` \| `"deleted"` \| `"renamed"` \| `"copied"` | | +| `old_path` | string? | Absent for an added file. | +| `new_path` | string? | Absent for a deleted file. Show `new_path ?? old_path`. | +| `language` | string? | The language tag the path names, as Diffy names grammars: the extension (`rs`, `ts`, `py`) or, for a file without one, its lowercased name (`dockerfile`). Absent when the path names none, or when `fallback.kind` is `no_grammar`. | +| `binary` | boolean? | A binary file has no `hunks` and no `gap_after`. | +| `additions` | number | Added lines. | +| `deletions` | number | Removed lines. | +| `old_missing_newline` | boolean? | The old file's last line has no newline. | +| `new_missing_newline` | boolean? | The new file's last line has no newline. | +| `fallback` | Fallback? | Present when the file got a line diff instead of a semantic one. | +| `hunks` | Hunk[]? | In file order. | +| `gap_after` | Gap? | Unchanged lines after the last hunk. For a file with no hunks (unchanged, or renamed without edits) this is the whole file. | +| `outline` | OutlineItem[]? | Items the semantic diff found changed. | +| `moves` | Move[]? | Code the semantic diff found moved. | +| `renames` | Rename[]? | Identifiers the semantic diff found renamed. | + +A file renders as: for each hunk, its `gap_before` (if any), its header, +its rows; then `gap_after` (if any). + +### Fallback + +| Field | Type | | +| --- | --- | --- | +| `kind` | `"no_grammar"` \| `"grammar_unavailable"` \| `"semantic"` \| `"request_budget"` | `no_grammar`: no language is known for the path. `grammar_unavailable`: the path names a language whose grammar could not load. `semantic`: the semantic diff gave up, see `reason`. `request_budget`: the request's deadline passed before this file was diffed. | +| `reason` | `"too_large"` \| `"parse_errors"` \| `"error_coverage"` \| `"unparsed"` \| `"budget"` \| `"timeout"`? | Present when `kind` is `semantic`. The file was too large; had too many parse errors; parse errors covered too much of it; the parser gave up; the work budget ran out; the time budget ran out. | +| `message` | string | For people, e.g. `"exceeded the semantic time budget"`. Wording may change; branch on `kind` and `reason`. | + +Line-diff files still get changed-word spans on `modified` rows. + +## Hunk + +| Field | Type | | +| --- | --- | --- | +| `old_start`, `old_len`, `new_start`, `new_len` | number | The numbers of a unified diff's `@@ -old_start,old_len +new_start,new_len @@` line. A start is the side's first line in the hunk, or the line before it when that side is empty (`0` at the top of the file). | +| `header` | string? | The item enclosing the hunk's first change, for the hunk's header line: an item name from the semantic outline (`render_preview`), or a scope label from the grammar (`fn render_preview`, `impl Parser`, `class Parser`, `parse()`). Absent when neither is known. | +| `gap_before` | Gap? | Unchanged lines between the previous hunk (or the top of the file) and this one. | +| `rows` | Row[] | In display order; each side's line numbers ascend. | + +## Gap + +Unchanged lines left out of the hunks. The lines are identical on both +sides; the starts differ when earlier changes shifted lines. + +| Field | Type | | +| --- | --- | --- | +| `old_start` | number | First line on the old side. | +| `new_start` | number | First line on the new side. | +| `len` | number | Lines in the gap, at least 1. | + +To expand, send the server a Gap: the whole gap, or any sub-range of it +(the first 20 lines: `{old_start, new_start, len: 20}`; the last 20: +`{old_start: old_start + len - 20, new_start: new_start + len - 20, len: +20}`). The server answers with `context` rows (below), with syntax spans +when the request asked for syntax. The answer can be shorter than `len` +when the range runs past the end of the file or no longer matches it. + +## Row + +A row is one of four kinds, told apart by `kind`. + +### `context` + +An unchanged line. + +| Field | Type | | +| --- | --- | --- | +| `kind` | `"context"` | | +| `old_line` | number | | +| `new_line` | number | | +| `text` | string | | +| `syntax` | SyntaxSpan[]? | From the new file. | + +### `removed`, `added`, `modified` + +| Kind | Fields | +| --- | --- | +| `removed` | `old`: Line | +| `added` | `new`: Line | +| `modified` | `old`: Line, `new`: Line — a removed line beside the added line it became. | + +Split view: `old` goes left, `new` right; a `removed` row leaves the right +empty and an `added` row the left. Unified view: a `modified` row is its +`old` line as removed then its `new` line as added; to read like `git +diff`, show a run of consecutive `modified` rows as all their old lines, +then all their new lines. + +### Line + +One side of a changed row. + +| Field | Type | | +| --- | --- | --- | +| `line` | number | Line number on its side. | +| `text` | string | | +| `changes` | ChangeSpan[]? | What changed within the line. Sorted, never overlapping. | +| `syntax` | SyntaxSpan[]? | From this side's file. | + +A `removed` or `added` line without `changes` changed as a whole. A +`modified` line without `changes` has no word-level detail: the lines are +too different to compare word by word, too long, or differ only in what +`text` does not show (a `\r`, an invalid byte, the final newline). + +### ChangeSpan + +| Field | Type | | +| --- | --- | --- | +| `start`, `end` | number | UTF-16 offsets within the line's `text`. | +| `intensity` | `"novel"` \| `"novel_word"` \| `"unchanged_context"` \| `"renamed"` | `novel`: syntax that exists only on this side (an inserted or deleted expression); `novel_word`: a changed word in an otherwise matched line; `unchanged_context`: unchanged syntax inside a changed line, for a subdued highlight or none; `renamed`: an identifier matched as renamed (see `renames`). | + +Line-diff files only use `novel_word`. + +### SyntaxSpan + +Present only when the request asked for syntax, the file's language has a +grammar, and the side is not too large. Text outside every span is plain. + +| Field | Type | | +| --- | --- | --- | +| `start`, `end` | number | UTF-16 offsets within the line's `text`. Sorted, never overlapping. | +| `class` | string | One of `keyword`, `string`, `comment`, `number`, `type`, `function`, `operator`, `punctuation`, `variable`, `constant`, `builtin`, `attribute`, `tag`, `property`, `namespace`, `label`, `preprocessor`. Map each to a theme colour; treat any other value as plain. | + +Syntax comes from parsing each whole file, so a line inside a multi-line +string or comment is highlighted as such. Changes and syntax are +independent layers: draw the change background, then the syntax colour. + +## OutlineItem + +| Field | Type | | +| --- | --- | --- | +| `side` | `"old"` \| `"new"` | Which file `line` refers to: `removed` items are on the old side, the rest on the new. | +| `line` | number | The item's first line. | +| `kind` | string | The grammar's node kind, e.g. `function_item`, `class_declaration`. | +| `name` | string | | +| `change` | `"added"` \| `"removed"` \| `"renamed"` \| `"moved"` \| `"edited"` | `renamed`: only identifiers inside it were renamed. | +| `edits` | number? | Changed lines of an `edited` item. | + +New-side items come first, then old-side ones. + +## Move + +`old_len` lines starting at old line `old_start` moved to `new_len` lines +starting at new line `new_start`. + +| Field | Type | | +| --- | --- | --- | +| `old_start`, `old_len`, `new_start`, `new_len` | number | | +| `from` | string? | Name of the item the code left. | +| `into` | string? | Name of the item the code arrived in. | +| `item` | string? | Name of the moved item itself, when the code is one. | + +## Rename + +| Field | Type | | +| --- | --- | --- | +| `old`, `new` | string | The identifier before and after. | +| `uses` | number | Occurrences renamed. | + +## TypeScript + +```ts +type Document = { version: 1; files: File[] }; + +type File = { + status: "modified" | "added" | "deleted" | "renamed" | "copied"; + old_path?: string; + new_path?: string; + language?: string; + binary?: boolean; + additions: number; + deletions: number; + old_missing_newline?: boolean; + new_missing_newline?: boolean; + fallback?: Fallback; + hunks?: Hunk[]; + gap_after?: Gap; + outline?: OutlineItem[]; + moves?: Move[]; + renames?: Rename[]; +}; + +type Fallback = { + kind: "no_grammar" | "grammar_unavailable" | "semantic" | "request_budget"; + reason?: "too_large" | "parse_errors" | "error_coverage" | "unparsed" | "budget" | "timeout"; + message: string; +}; + +type Hunk = { + old_start: number; + old_len: number; + new_start: number; + new_len: number; + header?: string; + gap_before?: Gap; + rows: Row[]; +}; + +type Gap = { old_start: number; new_start: number; len: number }; + +type Row = + | { kind: "context"; old_line: number; new_line: number; text: string; syntax?: SyntaxSpan[] } + | { kind: "removed"; old: Line } + | { kind: "added"; new: Line } + | { kind: "modified"; old: Line; new: Line }; + +type Line = { line: number; text: string; changes?: ChangeSpan[]; syntax?: SyntaxSpan[] }; + +type ChangeSpan = { + start: number; // UTF-16 + end: number; + intensity: "novel" | "novel_word" | "unchanged_context" | "renamed"; +}; + +type SyntaxSpan = { start: number; end: number; class: string }; // UTF-16 + +type OutlineItem = { + side: "old" | "new"; + line: number; + kind: string; + name: string; + change: "added" | "removed" | "renamed" | "moved" | "edited"; + edits?: number; +}; + +type Move = { + old_start: number; + old_len: number; + new_start: number; + new_len: number; + from?: string; + into?: string; + item?: string; +}; + +type Rename = { old: string; new: string; uses: number }; +``` + +## Example + +A Rust file where `render` became `render_preview` and an argument +changed after a line with non-ASCII text, diffed semantically, without +syntax. Lines 1–9 and 14–20 are unchanged and collapsed. `é` is one UTF-16 +unit and `😀` two, so the changed `2` starts at 16 (byte 19). + +```json +{ + "version": 1, + "files": [ + { + "status": "modified", + "old_path": "src/view.rs", + "new_path": "src/view.rs", + "language": "rs", + "additions": 2, + "deletions": 2, + "hunks": [ + { + "old_start": 10, + "old_len": 4, + "new_start": 10, + "new_len": 4, + "header": "render_preview", + "gap_before": { "old_start": 1, "new_start": 1, "len": 9 }, + "rows": [ + { "kind": "context", "old_line": 10, "new_line": 10, "text": "" }, + { + "kind": "modified", + "old": { + "line": 11, + "text": "fn render() {", + "changes": [{ "start": 3, "end": 9, "intensity": "renamed" }] + }, + "new": { + "line": 11, + "text": "fn render_preview() {", + "changes": [{ "start": 3, "end": 17, "intensity": "renamed" }] + } + }, + { + "kind": "modified", + "old": { + "line": 12, + "text": " draw(\"é😀\", 1);", + "changes": [{ "start": 16, "end": 17, "intensity": "novel_word" }] + }, + "new": { + "line": 12, + "text": " draw(\"é😀\", 2);", + "changes": [{ "start": 16, "end": 17, "intensity": "novel_word" }] + } + }, + { "kind": "context", "old_line": 13, "new_line": 13, "text": "}" } + ] + } + ], + "gap_after": { "old_start": 14, "new_start": 14, "len": 7 }, + "outline": [ + { + "side": "new", + "line": 11, + "kind": "function_item", + "name": "render_preview", + "change": "edited", + "edits": 2 + } + ], + "renames": [{ "old": "render", "new": "render_preview", "uses": 1 }] + }, + { + "status": "added", + "new_path": "notes.txt", + "language": "txt", + "additions": 1, + "deletions": 0, + "new_missing_newline": true, + "hunks": [ + { + "old_start": 0, + "old_len": 0, + "new_start": 1, + "new_len": 1, + "rows": [{ "kind": "added", "new": { "line": 1, "text": "hello" } }] + } + ] + } + ] +} +``` + +With syntax requested, rows also carry spans, e.g. the old line 12 above +gains: + +```json +"syntax": [ + { "start": 4, "end": 8, "class": "function" }, + { "start": 9, "end": 14, "class": "string" }, + { "start": 16, "end": 17, "class": "constant" } +] +``` diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs index 249eaed6..31a4cdf7 100644 --- a/crates/diffy-compare/src/wire.rs +++ b/crates/diffy-compare/src/wire.rs @@ -1 +1,996 @@ -//! The versioned display format; filled in by the wire-format change. +//! The versioned display format that non-Rust clients render diffs from. +//! +//! [`file()`] and [`document`] project compare results ([`FileDiff`]) into +//! plain serializable values; [`expand`] produces the rows of a collapsed +//! gap on request. `FORMAT.md` beside this crate's manifest is the +//! client-facing spec of every field. +//! +//! Versioning: [`Document::version`] is [`FORMAT_VERSION`]. It is bumped +//! for any breaking change: removing or renaming a field or enum value, +//! changing a field's type, meaning, or coordinates, or making an optional +//! field required. Adding an optional field, or a value to an enum that +//! clients are told to tolerate (syntax classes), is not breaking and +//! keeps the version. +//! +//! Coordinates: line numbers are one-based. Span offsets are UTF-16 code +//! units within the row's text, the way JavaScript indexes strings, and +//! ranges are half-open. Text is each line as [`TextStore::display_line`] +//! shows it: no newline, no trailing `\r`, invalid UTF-8 bytes as U+FFFD. +//! +//! Rows are layout-neutral: a `modified` row carries the removed line and +//! the added line it became, so split and unified layouts both render +//! without recomputing anything. + +use std::borrow::Cow; +use std::cell::OnceCell; +use std::ops::Range; + +use diffy_diff::{ + AlignedRow, BlockKind, ChangeIntensity, DiffDocument, FileStatus, ItemChange, Side as DocSide, + TextStore, inline_diff, +}; +use diffy_syntax::{GrammarStore, HighlightKind, HighlightSpan, LanguageId, ScopeTable}; +use serde::{Deserialize, Serialize}; + +use crate::{FileDiff, FileInput}; + +/// The format version every [`Document`] carries. +pub const FORMAT_VERSION: u32 = 1; + +/// Sides longer than this get no syntax spans or scope headers by default. +pub const SYNTAX_MAX_BYTES: usize = 1 << 20; + +/// What a projection adds beyond the diff itself. +#[derive(Debug, Clone, Copy)] +pub struct WireOptions<'a> { + /// Grammars for syntax spans and for hunk headers of files without a + /// semantic outline. `None` (the default) emits neither. + pub syntax: Option<&'a GrammarStore>, + /// Sides longer than this, in bytes, get no syntax spans or scope + /// headers. + pub syntax_max_bytes: usize, +} + +impl Default for WireOptions<'_> { + fn default() -> Self { + Self { + syntax: None, + syntax_max_bytes: SYNTAX_MAX_BYTES, + } + } +} + +// ---- Format ------------------------------------------------------------ + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Document { + /// [`FORMAT_VERSION`] of the producer. + pub version: u32, + pub files: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct File { + pub status: Status, + /// `None` for an added file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub old_path: Option, + /// `None` for a deleted file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_path: Option, + /// The language tag the path names (`rs`, `ts`, `dockerfile`), as + /// diffy names grammars; `None` when it names none or the compare + /// found no grammar for it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub language: Option, + /// A binary file has no hunks or gaps. + #[serde(default, skip_serializing_if = "is_false")] + pub binary: bool, + pub additions: u32, + pub deletions: u32, + /// The old side's last line has no newline. + #[serde(default, skip_serializing_if = "is_false")] + pub old_missing_newline: bool, + /// The new side's last line has no newline. + #[serde(default, skip_serializing_if = "is_false")] + pub new_missing_newline: bool, + /// Why the file got a line diff instead of a semantic one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hunks: Vec, + /// Unchanged lines after the last hunk; for a file without hunks, the + /// whole file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_after: Option, + /// Items the semantic diff found changed: new-side items, then + /// old-side ones. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outline: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub moves: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub renames: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Status { + Modified, + Added, + Deleted, + Renamed, + Copied, +} + +impl From for Status { + fn from(status: FileStatus) -> Self { + match status { + FileStatus::Modified => Self::Modified, + FileStatus::Added => Self::Added, + FileStatus::Deleted => Self::Deleted, + FileStatus::Renamed => Self::Renamed, + FileStatus::Copied => Self::Copied, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Fallback { + pub kind: FallbackKind, + /// Set when `kind` is `semantic`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// For people; wording may change between releases. + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FallbackKind { + NoGrammar, + GrammarUnavailable, + Semantic, + RequestBudget, +} + +/// Why the semantic diff gave up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SemanticReason { + TooLarge, + ParseErrors, + ErrorCoverage, + Unparsed, + Budget, + Timeout, +} + +impl From<&crate::Fallback> for Fallback { + fn from(fallback: &crate::Fallback) -> Self { + use diffy_semantic::Fallback as Semantic; + let (kind, reason) = match fallback { + crate::Fallback::NoGrammar => (FallbackKind::NoGrammar, None), + crate::Fallback::GrammarUnavailable(_) => (FallbackKind::GrammarUnavailable, None), + crate::Fallback::RequestBudget => (FallbackKind::RequestBudget, None), + crate::Fallback::Semantic(semantic) => { + let reason = match semantic { + Semantic::TooLarge(_) => SemanticReason::TooLarge, + Semantic::ParseErrors(_) => SemanticReason::ParseErrors, + Semantic::ErrorCoverage(_) => SemanticReason::ErrorCoverage, + Semantic::Unparsed => SemanticReason::Unparsed, + Semantic::Budget => SemanticReason::Budget, + Semantic::Timeout => SemanticReason::Timeout, + }; + (FallbackKind::Semantic, Some(reason)) + } + }; + Self { + kind, + reason, + message: fallback.to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Hunk { + /// The `@@ -old_start,old_len +new_start,new_len @@` numbers: a start + /// is the first line, or the line before an empty side. + pub old_start: u32, + pub old_len: u32, + pub new_start: u32, + pub new_len: u32, + /// The item enclosing the hunk's first change (`render_preview`, or a + /// scope label such as `impl Parser`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub header: Option, + /// Unchanged lines between the previous hunk (or the file start) and + /// this one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gap_before: Option, + pub rows: Vec, +} + +/// Collapsed unchanged lines, the same on both sides. Also the request +/// for [`expand`]: any sub-range of a gap is a valid `Gap`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Gap { + /// One-based first line on each side. + pub old_start: u32, + pub new_start: u32, + pub len: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Row { + /// An unchanged line. `syntax` is the new side's. + Context { + old_line: u32, + new_line: u32, + text: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + syntax: Vec, + }, + Removed { + old: Line, + }, + Added { + new: Line, + }, + /// A removed line beside the added line it became. + Modified { + old: Line, + new: Line, + }, +} + +/// One side's line of a changed row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Line { + /// One-based line number on its side. + pub line: u32, + pub text: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub changes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub syntax: Vec, +} + +/// A changed run of a line. Offsets are UTF-16 code units within the +/// line's text; spans are sorted and disjoint. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChangeSpan { + pub start: u32, + pub end: u32, + pub intensity: Intensity, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Intensity { + /// Syntax that exists only on this side. + Novel, + /// A changed word inside an otherwise matched line. + NovelWord, + /// Unchanged syntax inside a changed line. + UnchangedContext, + /// An identifier the diff matched as renamed (see [`File::renames`]). + Renamed, +} + +impl From for Intensity { + fn from(intensity: ChangeIntensity) -> Self { + match intensity { + ChangeIntensity::Novel => Self::Novel, + ChangeIntensity::NovelWord => Self::NovelWord, + ChangeIntensity::UnchangedContext => Self::UnchangedContext, + ChangeIntensity::Renamed => Self::Renamed, + } + } +} + +/// A highlighted run of a line. Offsets are UTF-16 code units within the +/// line's text; spans are sorted and disjoint, and text outside them is +/// plain. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SyntaxSpan { + pub start: u32, + pub end: u32, + /// A [`HighlightKind::name`] such as `keyword` or `string`; clients + /// map it to their theme and treat unknown classes as plain. + pub class: Cow<'static, str>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Side { + Old, + New, +} + +impl From for Side { + fn from(side: DocSide) -> Self { + match side { + DocSide::Old => Self::Old, + DocSide::New => Self::New, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutlineItem { + pub side: Side, + /// One-based first line of the item on `side`. + pub line: u32, + /// The grammar's item kind, such as `function_item`. + pub kind: String, + pub name: String, + pub change: ItemChangeKind, + /// Changed lines of an `edited` item. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edits: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ItemChangeKind { + Added, + Removed, + Renamed, + Moved, + Edited, +} + +/// Code that moved: `old_len` lines from `old_start` became `new_len` +/// lines at `new_start` (one-based). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Move { + pub old_start: u32, + pub old_len: u32, + pub new_start: u32, + pub new_len: u32, + /// The item the code left. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from: Option, + /// The item the code arrived in. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub into: Option, + /// The moved item itself, when the code is one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub item: Option, +} + +/// An identifier renamed throughout the file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Rename { + pub old: String, + pub new: String, + pub uses: u32, +} + +fn is_false(value: &bool) -> bool { + !*value +} + +// ---- Projection -------------------------------------------------------- + +/// Projects every file; `inputs` and `diffs` pair up by index, and extra +/// entries of the longer slice are ignored. +pub fn document(inputs: &[FileInput], diffs: &[FileDiff], options: &WireOptions<'_>) -> Document { + Document { + version: FORMAT_VERSION, + files: inputs + .iter() + .zip(diffs) + .map(|(input, diff)| file(input, diff, options)) + .collect(), + } +} + +/// Projects one file. Highlighting and scope headers parse each side once; +/// everything else is linear in the rows shown. +pub fn file(input: &FileInput, diff: &FileDiff, options: &WireOptions<'_>) -> File { + let doc = &diff.document; + let has_file = doc.file_count() > 0; + let binary = crate::is_binary(&input.old) + || crate::is_binary(&input.new) + || doc.files().meta.first().is_some_and(|meta| meta.binary); + let (additions, deletions) = if has_file { + doc.stats(0).known().unwrap_or((0, 0)) + } else { + (0, 0) + }; + let language = match diff.fallback { + Some(crate::Fallback::NoGrammar) => None, + _ => LanguageId::from_path(input.display_path()).map(|id| id.as_str().to_owned()), + }; + let mut out = File { + status: input.status.into(), + old_path: input.old_path.clone(), + new_path: input.new_path.clone(), + language, + binary, + additions, + deletions, + old_missing_newline: false, + new_missing_newline: false, + fallback: diff.fallback.as_ref().map(Fallback::from), + hunks: Vec::new(), + gap_after: None, + outline: Vec::new(), + moves: Vec::new(), + renames: Vec::new(), + }; + if binary || !has_file { + return out; + } + let facts = doc.facts(0); + out.old_missing_newline = facts.old_missing_newline; + out.new_missing_newline = facts.new_missing_newline; + if let Some(semantics) = doc.semantics(0) { + out.outline = semantics.outline().iter().map(outline_item).collect(); + out.moves = semantics.moves().iter().map(wire_move).collect(); + out.renames = semantics + .renames() + .iter() + .map(|rename| Rename { + old: rename.old.to_string(), + new: rename.new.to_string(), + uses: rename.uses, + }) + .collect(); + } + Projector::new(doc, input, options).hunks(&mut out); + out +} + +/// Context rows for `gap` (or any sub-range of a gap) of `input`, with +/// the new side's syntax spans when `options` asks for them. Highlighting +/// parses the whole new side so lexical state is right. Rows stop early +/// where the range leaves either side or the sides differ, as a stale +/// request would. +pub fn expand(input: &FileInput, gap: &Gap, options: &WireOptions<'_>) -> Vec { + if gap.len == 0 + || gap.old_start == 0 + || gap.new_start == 0 + || crate::is_binary(&input.old) + || crate::is_binary(&input.new) + { + return Vec::new(); + } + let old = TextStore::from_bytes(&input.old); + let new = TextStore::from_bytes(&input.new); + let (old_first, new_first) = (gap.old_start - 1, gap.new_start - 1); + let len = gap + .len + .min(old.line_count().saturating_sub(old_first)) + .min(new.line_count().saturating_sub(new_first)); + let new_path = input.new_path.as_deref().or(input.old_path.as_deref()); + let syntax = highlights(options, new_path, &new); + let mut rows = Vec::with_capacity(len as usize); + for k in 0..len { + let (o, n) = (old_first + k, new_first + k); + if old.line(o) != new.line(n) { + break; + } + rows.push(context_row(&new, &syntax, n, o + 1, n + 1)); + } + rows +} + +fn outline_item(item: &diffy_diff::OutlineItem) -> OutlineItem { + let (change, edits) = match item.change { + ItemChange::Added => (ItemChangeKind::Added, None), + ItemChange::Removed => (ItemChangeKind::Removed, None), + ItemChange::Renamed => (ItemChangeKind::Renamed, None), + ItemChange::Moved => (ItemChangeKind::Moved, None), + ItemChange::Edited(lines) => (ItemChangeKind::Edited, Some(lines)), + }; + OutlineItem { + side: item.side.into(), + line: item.line + 1, + kind: item.kind.to_string(), + name: item.name.to_string(), + change, + edits, + } +} + +fn wire_move(moved: &diffy_diff::Move) -> Move { + let name = |name: &Option>| name.as_deref().map(str::to_owned); + Move { + old_start: moved.old_line + 1, + old_len: moved.old_lines, + new_start: moved.new_line + 1, + new_len: moved.lines, + from: name(&moved.from), + into: name(&moved.into), + item: name(&moved.item), + } +} + +/// Builds the hunks of file 0 of a one-file document. +struct Projector<'a> { + doc: &'a DiffDocument, + input: &'a FileInput, + options: &'a WireOptions<'a>, + /// The stores hold only the lines the patch shows: no gaps, and no + /// syntax (fragments would parse out of context). + partial: bool, + texts: [&'a TextStore; 2], + syntax: [Vec; 2], + scopes: [OnceCell; 2], +} + +impl<'a> Projector<'a> { + fn new(doc: &'a DiffDocument, input: &'a FileInput, options: &'a WireOptions<'a>) -> Self { + let partial = doc.files().partial[0]; + let texts = [doc.text(0, DocSide::Old), doc.text(0, DocSide::New)]; + let syntax = if partial { + [Vec::new(), Vec::new()] + } else { + [DocSide::Old, DocSide::New] + .map(|side| highlights(options, side_path(input, side), texts[side as usize])) + }; + Self { + doc, + input, + options, + partial, + texts, + syntax, + scopes: [OnceCell::new(), OnceCell::new()], + } + } + + fn hunks(&self, out: &mut File) { + let (doc, partial) = (self.doc, self.partial); + let (h, b) = (doc.hunks(), doc.blocks()); + let structural = doc.prefers_structural(0); + let mut steps = Vec::new(); + // Zero-based ends of the previous hunk on each side. + let (mut old_end, mut new_end) = (0u32, 0u32); + for hunk in doc.files().hunks[0].clone() { + let hi = hunk as usize; + let old_lines = doc.hunk_lines(hunk, DocSide::Old); + let new_lines = doc.hunk_lines(hunk, DocSide::New); + let gap_before = if partial { + None + } else { + gap( + (old_end, new_end), + old_lines.start.saturating_sub(old_end), + new_lines.start.saturating_sub(new_end), + ) + }; + let rows_hint = h.old_len[hi].max(h.new_len[hi]) as usize; + let mut rows = Vec::with_capacity(rows_hint); + let aligned = if structural { + doc.aligned_rows(hunk) + } else { + &[] + }; + for block in h.blocks[hi].clone() { + let bi = block as usize; + let at = BlockAt { + old_store: b.old_store[bi], + new_store: b.new_store[bi], + old_line: b.old_line[bi] - 1, + new_line: b.new_line[bi] - 1, + }; + match b.kind[bi] { + BlockKind::Context => { + for k in 0..b.old_len[bi] { + rows.push(context_row( + self.texts[1], + &self.syntax[1], + at.new_store + k, + at.old_line + k + 1, + at.new_line + k + 1, + )); + } + } + BlockKind::Change => { + let lens = (b.old_len[bi], b.new_len[bi]); + steps.clear(); + if aligned.is_empty() || !structural_steps(aligned, &at, lens, &mut steps) { + steps.clear(); + positional_steps(lens, &mut steps); + } + for &step in &steps { + rows.push(self.change_row(&at, step)); + } + } + } + } + out.hunks.push(Hunk { + old_start: h.old_start[hi], + old_len: h.old_len[hi], + new_start: h.new_start[hi], + new_len: h.new_len[hi], + header: self.header(hunk), + gap_before, + rows, + }); + old_end = old_lines.end; + new_end = new_lines.end; + } + if !partial { + out.gap_after = gap( + (old_end, new_end), + self.texts[0].line_count().saturating_sub(old_end), + self.texts[1].line_count().saturating_sub(new_end), + ); + } + } + + /// The row of one step of a change block; offsets are within the block. + fn change_row(&self, at: &BlockAt, step: Step) -> Row { + let line = |side: DocSide, offset: u32| match side { + DocSide::Old => (at.old_store + offset, at.old_line + offset), + DocSide::New => (at.new_store + offset, at.new_line + offset), + }; + match step { + Step::Paired(o, n) => { + let (old, new) = (line(DocSide::Old, o), line(DocSide::New, n)); + let old_spans = self.doc.semantic_spans(0, DocSide::Old, old.1); + let new_spans = self.doc.semantic_spans(0, DocSide::New, new.1); + if old_spans.is_empty() && new_spans.is_empty() { + // A line-diff pair, or lines the semantic diff left + // unmarked: changed words come from the word diff. + let old_text = self.display(DocSide::Old, old.0); + let new_text = self.display(DocSide::New, new.0); + let words = inline_diff(old_text, new_text); + Row::Modified { + old: self.line(DocSide::Old, old, word_spans(&words.old)), + new: self.line(DocSide::New, new, word_spans(&words.new)), + } + } else { + Row::Modified { + old: self.line(DocSide::Old, old, semantic(old_spans)), + new: self.line(DocSide::New, new, semantic(new_spans)), + } + } + } + Step::Removed(o) => { + let old = line(DocSide::Old, o); + let spans = semantic(self.doc.semantic_spans(0, DocSide::Old, old.1)); + Row::Removed { + old: self.line(DocSide::Old, old, spans), + } + } + Step::Added(n) => { + let new = line(DocSide::New, n); + let spans = semantic(self.doc.semantic_spans(0, DocSide::New, new.1)); + Row::Added { + new: self.line(DocSide::New, new, spans), + } + } + } + } + + fn display(&self, side: DocSide, store: u32) -> &str { + self.texts[side as usize].display_line(store).unwrap_or("") + } + + /// One side's line: `(store index, zero-based line)`, and its changed + /// byte ranges within the display text. + fn line( + &self, + side: DocSide, + (store, line): (u32, u32), + changes: impl IntoIterator, Intensity)>, + ) -> Line { + let texts = self.texts[side as usize]; + let text = texts.display_line(store).unwrap_or(""); + let mut units = Utf16::new(text); + let changes = changes + .into_iter() + .filter_map(|(range, intensity)| { + let (start, end) = units.range(range)?; + Some(ChangeSpan { + start, + end, + intensity, + }) + }) + .collect(); + let syntax = syntax_spans(texts, &self.syntax[side as usize], store, &mut units); + Line { + line: line + 1, + text: text.to_owned(), + changes, + syntax, + } + } + + /// The item enclosing the hunk's first change: the nearest changed + /// item of the semantic outline starting at or before it, else the + /// innermost scope of the grammar's scope table. + fn header(&self, hunk: u32) -> Option { + let (h, b) = (self.doc.hunks(), self.doc.blocks()); + let hi = hunk as usize; + let first_change = h.blocks[hi] + .clone() + .map(|block| block as usize) + .find(|&bi| b.kind[bi] == BlockKind::Change); + // Zero-based anchors on each side, and the side the change shows. + let (old, new, side) = match first_change { + Some(bi) => ( + b.old_line[bi] - 1, + b.new_line[bi] - 1, + if b.new_len[bi] > 0 { + DocSide::New + } else { + DocSide::Old + }, + ), + None => ( + h.old_start[hi].saturating_sub(1), + h.new_start[hi].saturating_sub(1), + DocSide::New, + ), + }; + if let Some(semantics) = self.doc.semantics(0) { + // Items list no end line, so the nearest start before the + // change stands in for the enclosing item. Edited items sit on + // the new side, removed ones on the old; ties go to new. + let nearest = semantics + .outline() + .iter() + .filter_map(|item| { + let anchor = match item.side { + DocSide::Old => old, + DocSide::New => new, + }; + (item.line <= anchor).then(|| (anchor - item.line, item.side, item)) + }) + .min_by_key(|&(distance, side, _)| (distance, side == DocSide::Old)); + if let Some((_, _, item)) = nearest { + return Some(item.name.to_string()); + } + } + let line = match side { + DocSide::Old => old, + DocSide::New => new, + }; + // Sorted by start, longer first, so the last span containing the + // line is the innermost. + self.scopes(side) + .spans() + .iter() + .rfind(|span| span.lines.contains(&line)) + .map(|span| span.label.to_string()) + } + + fn scopes(&self, side: DocSide) -> &ScopeTable { + self.scopes[side as usize].get_or_init(|| { + let text = self.texts[side as usize].as_str(); + let language = side_path(self.input, side).and_then(LanguageId::from_path); + match (self.options.syntax, language) { + (Some(grammars), Some(language)) + if !self.partial && text.len() <= self.options.syntax_max_bytes => + { + diffy_syntax::scope_table(grammars, &language, text) + } + _ => ScopeTable::default(), + } + }) + } +} + +/// Where a block's first line sits: store indices and zero-based lines. +struct BlockAt { + old_store: u32, + new_store: u32, + old_line: u32, + new_line: u32, +} + +/// The gap after zero-based ends `(old, new)`; the sides agree on its +/// length in a valid document, the shorter wins otherwise. +fn gap((old, new): (u32, u32), old_len: u32, new_len: u32) -> Option { + let len = old_len.min(new_len); + (len > 0).then_some(Gap { + old_start: old + 1, + new_start: new + 1, + len, + }) +} + +fn side_path(input: &FileInput, side: DocSide) -> Option<&str> { + let (own, other) = match side { + DocSide::Old => (&input.old_path, &input.new_path), + DocSide::New => (&input.new_path, &input.old_path), + }; + own.as_deref().or(other.as_deref()) +} + +/// A row of a change block; offsets are within the block. +#[derive(Debug, Clone, Copy)] +enum Step { + Paired(u32, u32), + Removed(u32), + Added(u32), +} + +/// Pairs a change block's lines as the semantic diff aligned them, when +/// its rows cover the block in order; offsets within the block. +fn structural_steps( + rows: &[AlignedRow], + at: &BlockAt, + (old_len, new_len): (u32, u32), + steps: &mut Vec, +) -> bool { + let offset = |line: Option, first: u32, len: u32| { + line.and_then(|line| line.checked_sub(first)) + .filter(|&offset| offset < len) + }; + let (mut o, mut n) = (0, 0); + for row in rows { + let ro = offset(row.old, at.old_line, old_len); + let rn = offset(row.new, at.new_line, new_len); + if ro.is_some_and(|i| i != o) || rn.is_some_and(|i| i != n) { + return false; + } + steps.push(match (ro, rn) { + (Some(a), Some(b)) => Step::Paired(a, b), + (Some(a), None) => Step::Removed(a), + (None, Some(b)) => Step::Added(b), + (None, None) => continue, + }); + o += u32::from(ro.is_some()); + n += u32::from(rn.is_some()); + } + o == old_len && n == new_len +} + +/// The k-th removed line beside the k-th added line, then the rest. +fn positional_steps((old_len, new_len): (u32, u32), steps: &mut Vec) { + let paired = old_len.min(new_len); + steps.extend((0..paired).map(|k| Step::Paired(k, k))); + steps.extend((paired..old_len).map(Step::Removed)); + steps.extend((paired..new_len).map(Step::Added)); +} + +fn word_spans(ranges: &[Range]) -> impl Iterator, Intensity)> + '_ { + ranges.iter().map(|range| { + ( + range.start as usize..range.end as usize, + Intensity::NovelWord, + ) + }) +} + +fn semantic( + spans: &[diffy_diff::SemanticSpan], +) -> impl Iterator, Intensity)> + '_ { + spans.iter().map(|span| { + ( + span.start as usize..span.end as usize, + span.intensity.into(), + ) + }) +} + +/// A context row of store line `store` of `text`, with its syntax. +fn context_row( + text: &TextStore, + syntax: &[HighlightSpan], + store: u32, + old_line: u32, + new_line: u32, +) -> Row { + let line = text.display_line(store).unwrap_or(""); + let mut units = Utf16::new(line); + Row::Context { + old_line, + new_line, + text: line.to_owned(), + syntax: syntax_spans(text, syntax, store, &mut units), + } +} + +/// Highlights a whole side once, when `options` asks for syntax and the +/// side is within the size cap. +fn highlights( + options: &WireOptions<'_>, + path: Option<&str>, + text: &TextStore, +) -> Vec { + let source = text.as_str(); + match (options.syntax, path.and_then(LanguageId::from_path)) { + (Some(grammars), Some(language)) + if !source.is_empty() && source.len() <= options.syntax_max_bytes => + { + diffy_syntax::highlight(grammars, &language, source) + } + _ => Vec::new(), + } +} + +/// The spans of whole-text `syntax` on store line `store`, clipped to its +/// display text and converted to UTF-16 offsets within it. +fn syntax_spans( + text: &TextStore, + syntax: &[HighlightSpan], + store: u32, + units: &mut Utf16<'_>, +) -> Vec { + let Some(line) = text.display_range(store) else { + return Vec::new(); + }; + if syntax.is_empty() || line.is_empty() { + return Vec::new(); + } + // Spans are sorted and disjoint, so their ends ascend too. + let first = syntax.partition_point(|span| span.range().end <= line.start); + syntax[first..] + .iter() + .take_while(|span| (span.offset as usize) < line.end) + .filter(|span| span.kind != HighlightKind::Normal) + .filter_map(|span| { + let range = span.range(); + let start = range.start.max(line.start) - line.start; + let end = range.end.min(line.end) - line.start; + let (start, end) = units.range(start..end)?; + Some(SyntaxSpan { + start, + end, + class: Cow::Borrowed(span.kind.name()), + }) + }) + .collect() +} + +/// Converts byte offsets within a line to UTF-16 offsets, walking forward +/// from the previous offset; cheap for the ascending offsets of sorted +/// spans, and free for ASCII lines. +struct Utf16<'t> { + text: &'t str, + ascii: bool, + byte: usize, + unit: u32, +} + +impl<'t> Utf16<'t> { + fn new(text: &'t str) -> Self { + Self { + text, + ascii: text.is_ascii(), + byte: 0, + unit: 0, + } + } + + /// The UTF-16 offset of byte `at`, clamped to the text and moved back + /// to a char boundary. + fn at(&mut self, at: usize) -> u32 { + let mut at = at.min(self.text.len()); + if self.ascii { + return at as u32; + } + while !self.text.is_char_boundary(at) { + at -= 1; + } + if at < self.byte { + self.byte = 0; + self.unit = 0; + } + self.unit += self.text[self.byte..at].encode_utf16().count() as u32; + self.byte = at; + self.unit + } + + /// A byte range in UTF-16 offsets; `None` when it is empty. + fn range(&mut self, range: Range) -> Option<(u32, u32)> { + let start = self.at(range.start); + let end = self.at(range.end); + (start < end).then_some((start, end)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/diffy-compare/src/wire/tests.rs b/crates/diffy-compare/src/wire/tests.rs new file mode 100644 index 00000000..5e6b08aa --- /dev/null +++ b/crates/diffy-compare/src/wire/tests.rs @@ -0,0 +1,749 @@ +use std::sync::Arc; + +use diffy_diff::{ + AlignedRow, ChangeIntensity, DocumentBuilder, FileMeta, FileStatus, ItemChange, SemanticSpan, + Side as DocSide, SourceCoverage, TextStore, diff_texts, +}; + +use super::*; +use crate::{FileDiff, FileInput}; + +const PATH: &str = "src/lib.rs"; + +fn input(old: &str, new: &str) -> FileInput { + FileInput { + status: FileStatus::Modified, + old_path: Some(PATH.into()), + new_path: Some(PATH.into()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + } +} + +fn line_diff(old: &str, new: &str) -> (FileInput, FileDiff) { + let document = diff_texts(Some(PATH), Some(PATH), Some(old), Some(new), 3); + let diff = FileDiff { + document, + fallback: Some(crate::Fallback::NoGrammar), + }; + (input(old, new), diff) +} + +fn numbered(lines: u32) -> Vec { + (1..=lines).map(|i| format!("line {i}")).collect() +} + +fn text(lines: &[String]) -> String { + let mut out = lines.join("\n"); + out.push('\n'); + out +} + +fn meta() -> FileMeta { + FileMeta { + old_path: Some(Arc::from(PATH)), + new_path: Some(Arc::from(PATH)), + status: FileStatus::Modified, + ..FileMeta::default() + } +} + +/// The text a JavaScript client sees at UTF-16 range `start..end`. +fn utf16_slice(text: &str, start: u32, end: u32) -> String { + let units: Vec = text.encode_utf16().collect(); + String::from_utf16_lossy(&units[start as usize..end as usize]) +} + +fn context(old_line: u32, new_line: u32, text: &str) -> Row { + Row::Context { + old_line, + new_line, + text: text.into(), + syntax: Vec::new(), + } +} + +fn span(start: u32, end: u32, intensity: Intensity) -> ChangeSpan { + ChangeSpan { + start, + end, + intensity, + } +} + +fn modified(row: &Row) -> (&Line, &Line) { + match row { + Row::Modified { old, new } => (old, new), + other => panic!("expected a modified row, got {other:?}"), + } +} + +#[test] +fn line_diff_projects_hunks_gaps_and_word_changes() { + let old = numbered(30); + let mut new = old.clone(); + new[14] = "line fifteen".into(); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let file = file(&input, &diff, &WireOptions::default()); + + assert_eq!(file.status, Status::Modified); + assert_eq!(file.language.as_deref(), None, "no_grammar hides the tag"); + assert_eq!((file.additions, file.deletions), (1, 1)); + assert_eq!(file.hunks.len(), 1); + let hunk = &file.hunks[0]; + assert_eq!( + (hunk.old_start, hunk.old_len, hunk.new_start, hunk.new_len), + (12, 7, 12, 7) + ); + assert_eq!(hunk.header, None); + assert_eq!( + hunk.gap_before, + Some(Gap { + old_start: 1, + new_start: 1, + len: 11 + }) + ); + assert_eq!( + file.gap_after, + Some(Gap { + old_start: 19, + new_start: 19, + len: 12 + }) + ); + assert_eq!(hunk.rows.len(), 7); + assert_eq!(hunk.rows[0], context(12, 12, "line 12")); + assert_eq!(hunk.rows[6], context(18, 18, "line 18")); + let (old_line, new_line) = modified(&hunk.rows[3]); + assert_eq!((old_line.line, old_line.text.as_str()), (15, "line 15")); + assert_eq!( + (new_line.line, new_line.text.as_str()), + (15, "line fifteen") + ); + assert_eq!(old_line.changes, [span(5, 7, Intensity::NovelWord)]); + assert_eq!(new_line.changes, [span(5, 12, Intensity::NovelWord)]); +} + +#[test] +fn documents_round_trip_through_json_and_omit_empty_fields() { + let old = numbered(8); + let mut new = old.clone(); + new[3] = "line four".into(); + new.push("line 9".into()); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let doc = document( + std::slice::from_ref(&input), + std::slice::from_ref(&diff), + &WireOptions::default(), + ); + assert_eq!(doc.version, FORMAT_VERSION); + + let json = serde_json::to_string(&doc).expect("serializes"); + let back: Document = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(back, doc); + + let value: serde_json::Value = serde_json::from_str(&json).expect("parses"); + let file = &value["files"][0]; + assert_eq!(value["version"], 1); + assert_eq!(file["status"], "modified"); + assert_eq!( + file["fallback"], + serde_json::json!({ "kind": "no_grammar", "message": "no grammar for this file" }) + ); + for absent in [ + "binary", + "language", + "outline", + "moves", + "renames", + "gap_after", + "old_missing_newline", + ] { + assert!(file.get(absent).is_none(), "{absent} should be omitted"); + } + let rows = &file["hunks"][0]["rows"]; + assert_eq!( + rows[0], + serde_json::json!({ "kind": "context", "old_line": 1, "new_line": 1, "text": "line 1" }) + ); + assert_eq!(rows[3]["kind"], "modified"); + assert_eq!( + rows[3]["old"]["changes"][0], + serde_json::json!({ "start": 5, "end": 6, "intensity": "novel_word" }) + ); + assert!(rows[3]["old"].get("syntax").is_none()); + let last = &rows[rows.as_array().expect("rows").len() - 1]; + assert_eq!( + *last, + serde_json::json!({ "kind": "added", "new": { "line": 9, "text": "line 9" } }) + ); +} + +#[test] +fn span_offsets_count_utf16_units() { + let old = "let s = \"héllo 😀 world\";\n"; + let new = "let s = \"héllo 😀 there\";\n"; + let (input, diff) = line_diff(old, new); + let file = file(&input, &diff, &WireOptions::default()); + let (old_line, new_line) = modified(&file.hunks[0].rows[0]); + let [old_span] = old_line.changes[..] else { + panic!("one old span: {:?}", old_line.changes); + }; + let [new_span] = new_line.changes[..] else { + panic!("one new span: {:?}", new_line.changes); + }; + // `é` is one unit and `😀` two, so "world" starts at 18, not byte 21. + assert_eq!((old_span.start, old_span.end), (18, 23)); + assert_eq!( + utf16_slice(&old_line.text, old_span.start, old_span.end), + "world" + ); + assert_eq!( + utf16_slice(&new_line.text, new_span.start, new_span.end), + "there" + ); +} + +#[test] +fn invalid_utf8_lines_show_replacement_characters() { + let old = b"caf\xe9 au lait\n".to_vec(); + let new = b"caf\xe9 au chocolat\n".to_vec(); + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::from_bytes(&old), + TextStore::from_bytes(&new), + ); + file_builder.hunk(0, 0).change(1, 1); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let input = FileInput { + old, + new, + ..input("", "") + }; + let projected = file(&input, &diff, &WireOptions::default()); + let (old_line, new_line) = modified(&projected.hunks[0].rows[0]); + assert_eq!(old_line.text, "caf\u{FFFD} au lait"); + assert_eq!(old_line.changes, [span(8, 12, Intensity::NovelWord)]); + assert_eq!( + utf16_slice( + &new_line.text, + new_line.changes[0].start, + new_line.changes[0].end + ), + "chocolat" + ); +} + +/// A structural document: `render` renamed to `render_preview` with an +/// argument changed after multibyte text, plus an outline, a move, and a +/// rename. +fn semantic_diff() -> (FileInput, FileDiff) { + let old = "fn keep() {}\nfn render() {\n draw(\"é😀\", 1);\n}\n"; + let new = "fn keep() {}\nfn render_preview() {\n draw(\"é😀\", 2);\n}\n"; + let (old_store, new_store) = (TextStore::new(old), TextStore::new(new)); + let at = |store: &TextStore, line: u32, needle: &str| { + let text = store.display_line(line).expect("line"); + let start = text.find(needle).expect("needle") as u32; + (start, start + needle.len() as u32) + }; + let semantic = |(start, end), intensity| SemanticSpan { + start, + end, + intensity, + }; + let old_spans = [ + semantic(at(&old_store, 1, "render"), ChangeIntensity::Renamed), + semantic(at(&old_store, 2, "1"), ChangeIntensity::NovelWord), + ]; + let new_spans = [ + semantic( + at(&new_store, 1, "render_preview"), + ChangeIntensity::Renamed, + ), + semantic(at(&new_store, 2, "2"), ChangeIntensity::NovelWord), + ]; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file(meta(), SourceCoverage::Full, old_store, new_store); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 2) + .context(1); + for line in 0..4 { + file_builder.row(AlignedRow { + old: Some(line), + new: Some(line), + }); + } + for (line, (old_span, new_span)) in (1..).zip(old_spans.iter().zip(&new_spans)) { + file_builder.spans(DocSide::Old, line, std::slice::from_ref(old_span)); + file_builder.spans(DocSide::New, line, std::slice::from_ref(new_span)); + } + file_builder + .outline(vec![diffy_diff::OutlineItem { + side: DocSide::New, + line: 1, + kind: Arc::from("function_item"), + name: Arc::from("render_preview"), + change: ItemChange::Edited(2), + }]) + .renames(vec![diffy_diff::Rename { + old: Arc::from("render"), + new: Arc::from("render_preview"), + uses: 1, + }]) + .moves(vec![diffy_diff::Move { + old_line: 0, + old_lines: 1, + new_line: 0, + lines: 1, + from: None, + into: Some(Arc::from("render_preview")), + item: Some(Arc::from("keep")), + }]); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + (input(old, new), diff) +} + +#[test] +fn semantic_documents_keep_spans_outline_moves_and_renames() { + let (input, diff) = semantic_diff(); + let file = file(&input, &diff, &WireOptions::default()); + assert_eq!(file.language.as_deref(), Some("rs")); + assert_eq!(file.fallback, None); + assert_eq!( + file.outline, + [OutlineItem { + side: Side::New, + line: 2, + kind: "function_item".into(), + name: "render_preview".into(), + change: ItemChangeKind::Edited, + edits: Some(2), + }] + ); + assert_eq!( + file.renames, + [Rename { + old: "render".into(), + new: "render_preview".into(), + uses: 1, + }] + ); + assert_eq!( + file.moves, + [Move { + old_start: 1, + old_len: 1, + new_start: 1, + new_len: 1, + from: None, + into: Some("render_preview".into()), + item: Some("keep".into()), + }] + ); + + let hunk = &file.hunks[0]; + assert_eq!(hunk.header.as_deref(), Some("render_preview")); + assert_eq!((hunk.gap_before, file.gap_after), (None, None)); + assert_eq!(hunk.rows.len(), 4); + assert_eq!(hunk.rows[0], context(1, 1, "fn keep() {}")); + assert_eq!(hunk.rows[3], context(4, 4, "}")); + let (old_line, new_line) = modified(&hunk.rows[1]); + assert_eq!(old_line.changes, [span(3, 9, Intensity::Renamed)]); + assert_eq!(new_line.changes, [span(3, 17, Intensity::Renamed)]); + let (old_line, new_line) = modified(&hunk.rows[2]); + // Bytes 19..20, after `é` (2 bytes, 1 unit) and `😀` (4 bytes, 2 units). + assert_eq!(old_line.changes, [span(16, 17, Intensity::NovelWord)]); + assert_eq!(new_line.changes, [span(16, 17, Intensity::NovelWord)]); + assert_eq!(utf16_slice(&new_line.text, 16, 17), "2"); + + let json = serde_json::to_string(&file).expect("serializes"); + assert_eq!( + serde_json::from_str::(&json).expect("deserializes"), + file + ); + assert!(json.contains(r#""change":"edited","edits":2"#), "{json}"); +} + +#[test] +fn hunk_header_needs_an_item_before_the_change() { + let (input, diff) = semantic_diff(); + let mut document = DocumentBuilder::new(); + let source = diff.document.text(0, DocSide::Old).clone(); + let target = diff.document.text(0, DocSide::New).clone(); + let mut file_builder = document.file(meta(), SourceCoverage::Full, source, target); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 2) + .context(1) + .outline(vec![diffy_diff::OutlineItem { + side: DocSide::Old, + line: 3, + kind: Arc::from("function_item"), + name: Arc::from("later"), + change: ItemChange::Removed, + }]); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: document.build(), + fallback: None, + }; + let file = file(&input, &diff, &WireOptions::default()); + assert_eq!(file.hunks[0].header, None); +} + +#[test] +fn aligned_rows_pair_lines_where_the_semantic_diff_put_them() { + let old = "keep\none\ntwo\nend\n"; + let new = "keep\none!\ninserted\ntwo!\nend\n"; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::new(old), + TextStore::new(new), + ); + file_builder + .structural(true) + .hunk(0, 0) + .context(1) + .change(2, 3) + .context(1); + for (old, new) in [ + (Some(0), Some(0)), + (Some(1), Some(1)), + (None, Some(2)), + (Some(2), Some(3)), + (Some(3), Some(4)), + ] { + file_builder.row(AlignedRow { old, new }); + } + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let rows = &file(&input(old, new), &diff, &WireOptions::default()).hunks[0].rows; + let shape: Vec<(&str, Option, Option)> = rows + .iter() + .map(|row| match row { + Row::Context { + old_line, new_line, .. + } => ("context", Some(*old_line), Some(*new_line)), + Row::Removed { old } => ("removed", Some(old.line), None), + Row::Added { new } => ("added", None, Some(new.line)), + Row::Modified { old, new } => ("modified", Some(old.line), Some(new.line)), + }) + .collect(); + assert_eq!( + shape, + [ + ("context", Some(1), Some(1)), + ("modified", Some(2), Some(2)), + ("added", None, Some(3)), + ("modified", Some(3), Some(4)), + ("context", Some(4), Some(5)), + ] + ); + // Unmarked pairs still get the word diff. + let (_, new_line) = modified(&rows[1]); + assert_eq!(new_line.changes, [span(3, 4, Intensity::NovelWord)]); +} + +#[test] +fn gaps_track_each_side_and_expand_into_context_rows() { + let old = numbered(30); + let mut new = old.clone(); + new.splice(10..10, ["new a".to_string(), "new b".to_string()]); + let (input, diff) = line_diff(&text(&old), &text(&new)); + let file = file(&input, &diff, &WireOptions::default()); + let hunk = &file.hunks[0]; + assert_eq!( + (hunk.old_start, hunk.old_len, hunk.new_start, hunk.new_len), + (8, 6, 8, 8) + ); + let before = hunk.gap_before.expect("gap before"); + assert_eq!( + before, + Gap { + old_start: 1, + new_start: 1, + len: 7 + } + ); + let after = file.gap_after.expect("gap after"); + assert_eq!( + after, + Gap { + old_start: 14, + new_start: 16, + len: 17 + } + ); + + let options = WireOptions::default(); + let rows = expand(&input, &after, &options); + assert_eq!(rows.len(), 17); + assert_eq!(rows[0], context(14, 16, "line 14")); + assert_eq!(rows[16], context(30, 32, "line 30")); + + // The bottom three lines of the gap above the hunk. + let tail = Gap { + old_start: 5, + new_start: 5, + len: 3, + }; + let rows = expand(&input, &tail, &options); + assert_eq!( + rows, + [ + context(5, 5, "line 5"), + context(6, 6, "line 6"), + context(7, 7, "line 7"), + ] + ); + + // Past the end of the file: only the lines that exist. + let past = Gap { + old_start: 29, + new_start: 31, + len: 10, + }; + assert_eq!(expand(&input, &past, &options).len(), 2); + // Stale: the sides differ, so nothing is claimed unchanged. + let stale = Gap { + old_start: 14, + new_start: 14, + len: 3, + }; + assert!(expand(&input, &stale, &options).is_empty()); + let zero = Gap { + old_start: 0, + new_start: 0, + len: 3, + }; + assert!(expand(&input, &zero, &options).is_empty()); +} + +#[test] +fn unchanged_files_are_one_gap_and_added_files_have_none() { + let lines = text(&numbered(5)); + let (input, diff) = line_diff(&lines, &lines); + let file = file(&input, &diff, &WireOptions::default()); + assert!(file.hunks.is_empty()); + assert_eq!( + file.gap_after, + Some(Gap { + old_start: 1, + new_start: 1, + len: 5 + }) + ); + + let document = diff_texts(None, Some(PATH), None, Some("a\nb"), 3); + let input = FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some(PATH.into()), + old: Vec::new(), + new: b"a\nb".to_vec(), + }; + let diff = FileDiff { + document, + fallback: None, + }; + let file = super::file(&input, &diff, &WireOptions::default()); + assert_eq!(file.status, Status::Added); + assert!(file.new_missing_newline); + assert_eq!(file.gap_after, None); + let hunk = &file.hunks[0]; + assert_eq!(hunk.gap_before, None); + assert!(matches!( + &hunk.rows[..], + [Row::Added { .. }, Row::Added { .. }] + )); +} + +#[test] +fn binary_files_have_no_rows() { + let input = FileInput { + old: b"\0old".to_vec(), + new: b"\0new".to_vec(), + ..input("", "") + }; + let diff = FileDiff { + document: diff_texts(Some(PATH), Some(PATH), Some(""), Some(""), 3), + fallback: None, + }; + let file = file(&input, &diff, &WireOptions::default()); + assert!(file.binary); + assert!(file.hunks.is_empty() && file.gap_after.is_none()); + let json = serde_json::to_value(&file).expect("serializes"); + assert_eq!(json["binary"], true); + let gap = Gap { + old_start: 1, + new_start: 1, + len: 1, + }; + assert!(expand(&input, &gap, &WireOptions::default()).is_empty()); +} + +#[test] +fn fallbacks_serialize_as_stable_kinds() { + let cases = [ + ( + crate::Fallback::Semantic(diffy_semantic::Fallback::Timeout), + serde_json::json!({ + "kind": "semantic", + "reason": "timeout", + "message": "exceeded the semantic time budget", + }), + ), + ( + crate::Fallback::Semantic(diffy_semantic::Fallback::TooLarge(5)), + serde_json::json!({ + "kind": "semantic", + "reason": "too_large", + "message": "5 bytes exceeds the semantic byte limit", + }), + ), + ( + crate::Fallback::GrammarUnavailable("zig".into()), + serde_json::json!({ + "kind": "grammar_unavailable", + "message": "zig grammar unavailable", + }), + ), + ( + crate::Fallback::RequestBudget, + serde_json::json!({ + "kind": "request_budget", + "message": "request time budget exhausted", + }), + ), + ]; + for (fallback, expected) in cases { + let json = serde_json::to_value(Fallback::from(&fallback)).expect("serializes"); + assert_eq!(json, expected); + } +} + +/// Rust source whose `draw` call changes inside a method. +fn rust_change() -> (FileInput, FileDiff) { + let old = "fn helper() {}\n\nimpl Parser {\n fn render_preview(&self) {\n let s = \"é😀\";\n draw(1);\n }\n}\n"; + let new = old.replace("draw(1)", "draw(2)"); + line_diff(old, &new) +} + +#[test] +fn syntax_spans_are_classes_sliced_per_line_in_utf16() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let (input, diff) = rust_change(); + let options = WireOptions { + syntax: Some(&store), + ..WireOptions::default() + }; + let file = file(&input, &diff, &options); + let hunk = &file.hunks[0]; + assert_eq!(hunk.header.as_deref(), Some("fn render_preview")); + let Row::Context { text, syntax, .. } = &hunk.rows[2] else { + panic!("context row: {:?}", hunk.rows[2]); + }; + assert_eq!(text, " let s = \"é😀\";"); + let class_of = |slice: &str| { + syntax + .iter() + .find(|span| utf16_slice(text, span.start, span.end) == slice) + .map(|span| &*span.class) + }; + assert_eq!(class_of("let"), Some("keyword"), "{syntax:?}"); + assert_eq!(class_of("\"é😀\""), Some("string"), "{syntax:?}"); + let (old_line, _) = modified(&hunk.rows[3]); + let draw = old_line.syntax.first().expect("draw is highlighted"); + assert_eq!(utf16_slice(&old_line.text, draw.start, draw.end), "draw"); + assert_eq!(draw.class, "function"); + + // Expanded context is highlighted with the whole file's parse. + let gap = hunk.gap_before.expect("gap before"); + let rows = expand(&input, &gap, &options); + let Row::Context { syntax, .. } = &rows[0] else { + panic!("context row: {:?}", rows[0]); + }; + assert!(syntax.iter().any(|span| span.class == "keyword")); + + let capped = WireOptions { + syntax: Some(&store), + syntax_max_bytes: 8, + }; + let file = super::file(&input, &diff, &capped); + assert_eq!(file.hunks[0].header, None); + assert!(file.hunks[0].rows.iter().all(|row| match row { + Row::Context { syntax, .. } => syntax.is_empty(), + Row::Removed { old: line } | Row::Added { new: line } => line.syntax.is_empty(), + Row::Modified { old, new } => old.syntax.is_empty() && new.syntax.is_empty(), + })); +} + +#[test] +fn the_spec_example_matches_the_types() { + let spec = include_str!("../../FORMAT.md"); + let example = spec + .split("```json\n") + .nth(1) + .and_then(|rest| rest.split("```").next()) + .expect("FORMAT.md has a JSON example"); + let value: serde_json::Value = serde_json::from_str(example).expect("example parses"); + let document: Document = serde_json::from_value(value.clone()).expect("example is a Document"); + assert_eq!(document.version, FORMAT_VERSION); + // Nothing in the example is unknown to the types or spelled + // differently from what they write. + assert_eq!(serde_json::to_value(&document).expect("serializes"), value); +} + +#[test] +fn aligned_rows_that_miss_lines_fall_back_to_positional_pairs() { + let old = "one\ntwo\n"; + let new = "one!\ntwo!\n"; + let mut builder = DocumentBuilder::new(); + let mut file_builder = builder.file( + meta(), + SourceCoverage::Full, + TextStore::new(old), + TextStore::new(new), + ); + file_builder.structural(true).hunk(0, 0).change(2, 2); + // Only the first pair is aligned; the second line of each side is + // never mentioned. + file_builder.row(AlignedRow { + old: Some(0), + new: Some(0), + }); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: builder.build(), + fallback: None, + }; + let rows = &file(&input(old, new), &diff, &WireOptions::default()).hunks[0].rows; + assert_eq!(rows.len(), 2, "{rows:?}"); + for (row, number) in rows.iter().zip(1..) { + let (old_line, new_line) = modified(row); + assert_eq!((old_line.line, new_line.line), (number, number)); + } +} From b039e1641e3224a2faaa2f30a0fcf985eb46059f Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:16:59 +0000 Subject: [PATCH 03/16] feat(syntax): compile a fixed grammar set in with the bundled feature A headless consumer (the Node binding) needs grammars with no runtime packs and no download step. `syntax-pack bundle` builds and checks each language as a host pack from its pinned, hashed sources (which adapts its queries), compiles the same sources into a static library per grammar for the target, links them together once to catch clashing symbols, and writes bundle.tsv, the queries, and THIRD_PARTY_NOTICES.md. diffy-syntax's `bundled` feature reads that directory from DIFFY_SYNTAX_BUNDLE in a dependency-free build.rs, links the libraries, and generates the table; without the variable it builds an empty bundle with a cargo warning. `StoreConfig::bundled()` adds the grammars after local packs and before downloads; they share tagging, queries, the ABI check and catalog reporting with packs, through `Grammar::compile` now taking the language function. `bundled::languages()` lists them. The app does not enable the feature. --- crates/diffy-syntax/Cargo.toml | 13 +- crates/diffy-syntax/PACKS.md | 57 +++- crates/diffy-syntax/README.md | 16 +- crates/diffy-syntax/build.rs | 172 +++++++++++ crates/diffy-syntax/src/bundled.rs | 48 +++ crates/diffy-syntax/src/engine.rs | 80 +++-- crates/diffy-syntax/src/lib.rs | 22 +- crates/diffy-syntax/src/store.rs | 117 ++++++-- crates/diffy-syntax/src/tests.rs | 209 +++++++++++++ tools/syntax-pack/README.md | 27 ++ tools/syntax-pack/src/main.rs | 451 ++++++++++++++++++++++++++--- 11 files changed, 1097 insertions(+), 115 deletions(-) create mode 100644 crates/diffy-syntax/src/bundled.rs diff --git a/crates/diffy-syntax/Cargo.toml b/crates/diffy-syntax/Cargo.toml index df1aa589..dd49defd 100644 --- a/crates/diffy-syntax/Cargo.toml +++ b/crates/diffy-syntax/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "diffy-syntax" -description = "Tree-sitter syntax highlighting for Diffy, with grammars loaded at runtime from signed packs." +description = "Tree-sitter syntax highlighting for Diffy, with grammars loaded at runtime from signed packs or compiled in." version.workspace = true edition = "2024" rust-version = "1.92" license = "MIT OR Apache-2.0" publish = false -# No grammar is compiled in. Grammars arrive as packs (a shared library -# plus queries and a manifest) from a local directory the app names, or -# from a signed index the app configures; see the crate docs and -# PACKS.md. Without any feature this crate has no +# No grammar is compiled in unless `bundled` is on. Grammars arrive as +# packs (a shared library plus queries and a manifest) from a local +# directory the app names, or from a signed index the app configures; see +# the crate docs and PACKS.md. Without any feature this crate has no # dependencies and every lookup falls back to plain text. [dependencies] tree-sitter = { version = "0.26", optional = true } @@ -43,6 +43,9 @@ engine = [ ] # Downloading packs on demand from a signed index. download = ["engine", "dep:quark-update", "dep:ureq"] +# Grammars compiled in from the bundle `syntax-pack bundle` writes, named +# by $DIFFY_SYNTAX_BUNDLE at build time (empty, with a warning, without it). +bundled = ["engine"] [dev-dependencies] tempfile = "3" diff --git a/crates/diffy-syntax/PACKS.md b/crates/diffy-syntax/PACKS.md index 7c528911..3ac58ad4 100644 --- a/crates/diffy-syntax/PACKS.md +++ b/crates/diffy-syntax/PACKS.md @@ -2,7 +2,8 @@ Load tree-sitter grammars at runtime, from packs, to highlight diffs. -- No grammar is compiled into the app. +- No grammar is compiled into the app. The `bundled` feature compiles a + fixed set into other builds (see [Bundled grammars](#bundled-grammars)). - A pack is a shared library exporting the tree-sitter language function, plus its highlight query, an optional injection query, and a manifest. - A `GrammarStore` feeds the diff view; an app adapts it to Quark's @@ -16,6 +17,7 @@ Load tree-sitter grammars at runtime, from packs, to highlight diffs. |---|---|---|---| | Local pack roots | `StoreConfig::local_packs(root)` | `diffy-syntax/engine` | Unsigned; the app opts in | | Signed index | `StoreConfig::downloads(Downloads::new(app_id, url, &keys))` | `diffy-syntax/download` | Ed25519 signature over the index | +| Bundled grammars | `StoreConfig::bundled()` | `diffy-syntax/bundled` | Part of the binary | From [diff_demo.rs](../diffy-ui/examples/diff_demo.rs): @@ -26,8 +28,8 @@ let store = GrammarStore::new(StoreConfig::new().local_packs(root)); diff.enable_syntax(store); // DiffViewState ``` -- Search order: local roots (in the order added), then the download cache, - then the network. +- Search order: local roots (in the order added), then bundled grammars, + then the download cache, then the network. - No default root and no default index URL. - A pack root holds `//`, as `syntax-pack` writes it. @@ -66,12 +68,14 @@ A fence tag or file extension resolves once per store; the result is kept, so an unknown language costs one map lookup afterwards. 1. A local pack whose name, alias, or extension matches loads at once. -2. A pack in the cached signed index, with every file present and matching +2. With `StoreConfig::bundled`, a bundled grammar that matches loads at + once. +3. A pack in the cached signed index, with every file present and matching its SHA-256, loads at once. -3. Otherwise, with downloads configured, the language is pending. The fetch +4. Otherwise, with downloads configured, the language is pending. The fetch thread downloads the index (at most once per store, cached) and then the pack. -4. A language missing from the fresh index, or whose download fails, is +5. A language missing from the fresh index, or whose download fails, is unavailable until `GrammarStore::retry` (which refetches the index). - `HighlightWorker` answers a pending language with a plain result marked @@ -128,6 +132,45 @@ cargo run -p syntax-pack -- check # re-check built packs for thi - TypeScript and TSX also read JavaScript's queries from `DIR/javascript/`. +## Bundled grammars + +For builds that must highlight with no packs and no download step, such +as a headless server. The Diffy app does not use it. + +- Default set: C, C++, CSS, Go, HTML, JavaScript, JSON, Kotlin, Markdown + (with `markdown_inline`), Python, Rust, Swift, TOML, TSX, TypeScript, + YAML. +- `syntax-pack bundle` builds and checks each as a host pack (which adapts + its queries), then compiles the same verified sources into a static + library per grammar for the target and links them together once to + catch clashing symbols. +- Output, `//`: `lib/` (static libraries), `queries/`, + `bundle.tsv` (the table `build.rs` reads), `THIRD_PARTY_NOTICES.md`. +- Build diffy-syntax with `--features bundled` and `DIFFY_SYNTAX_BUNDLE` + set to the absolute ``; `build.rs` reads `//` and + links it. +- Without `DIFFY_SYNTAX_BUNDLE` the feature builds with an empty bundle and + a cargo warning. +- Bundled grammars get the same tags, queries, ABI check, and `catalog()` + entries as packs; nothing is loaded or hashed at run time. +- `diffy_syntax::bundled::languages()` lists what a build has. +- Ship `THIRD_PARTY_NOTICES.md` with anything you distribute: it holds each + grammar's license, repositories, commits, and license text, which MIT + requires. + +```sh +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" # default set, this host +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" rust go # just these +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" --target x86_64-apple-darwin +DIFFY_SYNTAX_BUNDLE="$PWD/target/syntax-bundle" cargo build -p my-server --features diffy-syntax/bundled +``` + +```rust +use diffy_syntax::{GrammarStore, StoreConfig}; + +let store = GrammarStore::new(StoreConfig::new().bundled()); +``` + Maintainer details (pinning, cross-target checks, publishing) are in [tools/syntax-pack/README.md](../../tools/syntax-pack/README.md). @@ -175,5 +218,7 @@ Loading a pack runs native code with the app's privileges. Full model: - **The cache** is not defended against anyone who can already write the user's files (including swapping a library between hash check and `dlopen`). +- **Bundled grammars** are part of the binary, built from the same pinned, + hashed sources; nothing is loaded at run time. - **Parsers** see untrusted text. A crash in a grammar's C code kills the process; a panic in highlighting is caught by the worker. diff --git a/crates/diffy-syntax/README.md b/crates/diffy-syntax/README.md index fd39c87f..9d4760ba 100644 --- a/crates/diffy-syntax/README.md +++ b/crates/diffy-syntax/README.md @@ -1,14 +1,16 @@ # diffy-syntax Tree-sitter syntax highlighting for [Diffy](../../README.md). -No grammar is compiled in: each language is a pack (a shared library, its -queries, and a manifest) that a `GrammarStore` loads from local pack -directories the app trusts or, with the `download` feature, from a signed -index. Code renders plain until its grammar is available. +No grammar is compiled in by default: each language is a pack (a shared +library, its queries, and a manifest) that a `GrammarStore` loads from local +pack directories the app trusts or, with the `download` feature, from a +signed index. Code renders plain until its grammar is available. Builds that +cannot load packs (a headless server) link a fixed set with `bundled`. -Features: `engine` (the tree-sitter runtime and local packs) and `download` -(fetching packs over HTTPS with Ed25519-verified indexes). Without `engine` -the crate has no dependencies and every lookup falls back to plain text. +Features: `engine` (the tree-sitter runtime and local packs), `download` +(fetching packs over HTTPS with Ed25519-verified indexes), and `bundled` +(grammars compiled in from `syntax-pack bundle`). Without `engine` the crate +has no dependencies and every lookup falls back to plain text. `highlight` runs synchronously; `HighlightWorker` runs on a background thread and drops requests a newer generation superseded, which suits code diff --git a/crates/diffy-syntax/build.rs b/crates/diffy-syntax/build.rs index 205268e9..43cebd19 100644 --- a/crates/diffy-syntax/build.rs +++ b/crates/diffy-syntax/build.rs @@ -1,3 +1,13 @@ +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +/// The directory `syntax-pack bundle --out` wrote, read by the `bundled` +/// feature. +const BUNDLE_VAR: &str = "DIFFY_SYNTAX_BUNDLE"; +/// First line of `//bundle.tsv`; `syntax-pack` writes it. +const HEADER: &str = "# diffy-syntax bundle 1"; +const QUERIES: [&str; 5] = ["highlights", "injections", "locals", "semantic", "tags"]; + // Packs are built per target triple, and Rust has no runtime constant for // the full triple (`std::env::consts` lacks the vendor and ABI parts that // tell gnu from musl, or msvc from gnu), so record the one this build is for. @@ -5,4 +15,166 @@ fn main() { let target = std::env::var("TARGET").expect("cargo sets TARGET for build scripts"); println!("cargo:rustc-env=DIFFY_SYNTAX_TARGET={target}"); println!("cargo:rerun-if-changed=build.rs"); + if std::env::var_os("CARGO_FEATURE_BUNDLED").is_some() + && let Err(error) = bundle(&target) + { + panic!("{BUNDLE_VAR}: {error}"); + } +} + +/// Writes `$OUT_DIR/bundled.rs`, the table `src/bundled.rs` includes, and +/// links the bundle's static libraries. +fn bundle(target: &str) -> Result<(), String> { + println!("cargo:rerun-if-env-changed={BUNDLE_VAR}"); + let out_dir = std::env::var_os("OUT_DIR").ok_or("cargo did not set OUT_DIR")?; + let code = match std::env::var_os(BUNDLE_VAR).filter(|value| !value.is_empty()) { + None => { + println!( + "cargo:warning={BUNDLE_VAR} is not set, so the `bundled` feature compiles no \ + grammar in; see crates/diffy-syntax/PACKS.md" + ); + "pub(crate) static GRAMMARS: &[Bundled] = &[];\n".to_owned() + } + Some(root) => { + let root = PathBuf::from(root); + if !root.is_absolute() { + return Err(format!("{} is not an absolute path", root.display())); + } + generate(&root.join(target), target)? + } + }; + let out = Path::new(&out_dir).join("bundled.rs"); + std::fs::write(&out, code).map_err(|e| format!("{}: {e}", out.display())) +} + +/// The table for the bundle in `dir`, one line of `bundle.tsv` per grammar: +/// language, symbol, ABI, library, aliases, extensions, source SHA-256 +/// (lists comma-separated, `-` when empty). Queries are +/// `queries//.scm`. +fn generate(dir: &Path, target: &str) -> Result { + let manifest = dir.join("bundle.tsv"); + let text = std::fs::read_to_string(&manifest).map_err(|e| { + format!( + "{}: {e}; write it with `syntax-pack bundle --target {target}`", + manifest.display() + ) + })?; + println!("cargo:rerun-if-changed={}", manifest.display()); + let mut lines = text.lines(); + if lines.next() != Some(HEADER) { + return Err(format!("{} is not a version 1 bundle", manifest.display())); + } + let lib_dir = dir.join("lib"); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + let mut externs = String::new(); + let mut table = String::new(); + let lines = lines.filter(|line| !line.is_empty() && !line.starts_with('#')); + for (index, line) in lines.enumerate() { + let bad = |what: &str| format!("{}: {what} in {line:?}", manifest.display()); + let fields: Vec<&str> = line.split('\t').collect(); + let [language, symbol, abi, library, aliases, extensions, source] = fields[..] else { + return Err(bad("not 7 tab-separated fields")); + }; + let list = |field: &str| -> Result, String> { + if field == "-" { + return Ok(Vec::new()); + } + field + .split(',') + .map(|tag| { + is_tag(tag) + .then(|| format!("{tag:?}")) + .ok_or_else(|| bad(&format!("bad tag {tag:?}"))) + }) + .collect() + }; + let (aliases, extensions) = (list(aliases)?, list(extensions)?); + if !is_tag(language) { + return Err(bad("bad language")); + } + if !is_c_identifier(symbol) { + return Err(bad("bad symbol")); + } + let abi: u32 = abi.parse().map_err(|_| bad("bad ABI"))?; + if library.is_empty() + || !library + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') + { + return Err(bad("bad library name")); + } + if source.len() != 64 || !source.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(bad("bad source hash")); + } + let archive = if target.contains("msvc") { + lib_dir.join(format!("{library}.lib")) + } else { + lib_dir.join(format!("lib{library}.a")) + }; + if !archive.is_file() { + return Err(format!("{} is missing", archive.display())); + } + println!("cargo:rerun-if-changed={}", archive.display()); + println!("cargo:rustc-link-lib=static={library}"); + + // Each grammar gets its own Rust name, so only the link name comes + // from the bundle. + let _ = writeln!( + externs, + " #[link_name = {symbol:?}]\n fn grammar_{index}() -> *const ();" + ); + let mut queries = String::new(); + for name in QUERIES { + let path = dir + .join("queries") + .join(language) + .join(format!("{name}.scm")); + let value = if path.is_file() { + println!("cargo:rerun-if-changed={}", path.display()); + let path = path + .to_str() + .ok_or_else(|| format!("{} is not UTF-8", path.display()))?; + if name == "highlights" { + format!("include_str!({path:?})") + } else { + format!("Some(include_str!({path:?}))") + } + } else if name == "highlights" { + return Err(format!("{} is missing", path.display())); + } else { + "None".to_owned() + }; + let _ = writeln!(queries, " {name}: {value},"); + } + let _ = write!( + table, + " Bundled {{\n name: {language:?},\n aliases: &[{}],\n \ + extensions: &[{}],\n abi: {abi},\n language: grammar_{index},\n \ + source: {source:?},\n{queries} }},\n", + aliases.join(", "), + extensions.join(", "), + ); + } + Ok(format!( + "// Generated by build.rs from {:?}.\nunsafe extern \"C\" {{\n{externs}}}\n\n\ + pub(crate) static GRAMMARS: &[Bundled] = &[\n{table}];\n", + manifest + )) +} + +/// A fence tag or extension, as `LanguageId::from_fence` accepts them. +fn is_tag(value: &str) -> bool { + !value.is_empty() + && value.len() <= 32 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'#' | b'.' | b'_' | b'-')) +} + +fn is_c_identifier(value: &str) -> bool { + let mut bytes = value.bytes(); + bytes + .next() + .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') } diff --git a/crates/diffy-syntax/src/bundled.rs b/crates/diffy-syntax/src/bundled.rs new file mode 100644 index 00000000..0d4d15f6 --- /dev/null +++ b/crates/diffy-syntax/src/bundled.rs @@ -0,0 +1,48 @@ +//! Grammars compiled into the binary with the `bundled` feature. +//! +//! `syntax-pack bundle` compiles a fixed set of grammars from their pinned +//! sources into static libraries and writes their queries and a table; +//! `build.rs` links them when `DIFFY_SYNTAX_BUNDLE` names that directory. +//! Without it the bundle is empty. A store uses them only after +//! [`crate::StoreConfig::bundled`]. + +/// One compiled-in grammar, as `build.rs` generates it. +pub(crate) struct Bundled { + pub(crate) name: &'static str, + pub(crate) aliases: &'static [&'static str], + pub(crate) extensions: &'static [&'static str], + /// The grammar's tree-sitter ABI, checked against the linked grammar. + pub(crate) abi: u32, + pub(crate) language: unsafe extern "C" fn() -> *const (), + /// SHA-256 of the pinned sources it was built from. + pub(crate) source: &'static str, + pub(crate) highlights: &'static str, + pub(crate) injections: Option<&'static str>, + pub(crate) locals: Option<&'static str>, + pub(crate) semantic: Option<&'static str>, + pub(crate) tags: Option<&'static str>, +} + +impl Bundled { + /// The canonical name, aliases, and extensions, each a lookup key. + pub(crate) fn tags(&self) -> impl Iterator { + std::iter::once(self.name) + .chain(self.aliases.iter().copied()) + .chain(self.extensions.iter().copied()) + } +} + +include!(concat!(env!("OUT_DIR"), "/bundled.rs")); + +/// The canonical names of the grammars compiled into this build; empty +/// when it was built without `DIFFY_SYNTAX_BUNDLE`. +pub fn languages() -> impl ExactSizeIterator + Clone { + GRAMMARS.iter().map(|grammar| grammar.name) +} + +/// The bundled grammar a name, alias, or extension names. +pub(crate) fn find(tag: &str) -> Option<&'static Bundled> { + GRAMMARS + .iter() + .find(|grammar| grammar.tags().any(|known| known.eq_ignore_ascii_case(tag))) +} diff --git a/crates/diffy-syntax/src/engine.rs b/crates/diffy-syntax/src/engine.rs index 4ededefb..f292d32f 100644 --- a/crates/diffy-syntax/src/engine.rs +++ b/crates/diffy-syntax/src/engine.rs @@ -67,13 +67,24 @@ impl Grammar { None => Ok(None), }; let injections = read(&manifest.injections)?; - let grammar = Self::compile( - &dir.join(&manifest.library.path), - &manifest.symbol, - Some(manifest.abi), - &highlights, - injections.as_deref(), - )?; + let library_path = dir.join(&manifest.library.path); + // SAFETY: loading a library runs its initializers, and calling the + // symbol runs its code. The pack passed its SHA-256 check against a + // manifest that is either signed (downloaded packs) or in a + // directory the app opted into (local packs); see the threat model + // in the crate docs. + let language = unsafe { + let library = libloading::Library::new(&library_path) + .map_err(|e| PackError::Library(e.to_string()))?; + let library: &'static libloading::Library = Box::leak(Box::new(library)); + let mut name = manifest.symbol.as_bytes().to_vec(); + name.push(0); + let symbol = library + .get:: *const ()>(&name) + .map_err(|e| PackError::Library(e.to_string()))?; + tree_sitter_language::LanguageFn::from_raw(*symbol) + }; + let grammar = Self::compile(language, manifest.abi, &highlights, injections.as_deref())?; Ok(Self { name: Some(Arc::from(manifest.language.as_str())), locals: read(&manifest.locals)?.map(Arc::from), @@ -84,6 +95,31 @@ impl Grammar { }) } + /// A grammar compiled into the binary, with its queries. + #[cfg(feature = "bundled")] + pub(crate) fn bundled(bundled: &crate::bundled::Bundled) -> Result { + // SAFETY: the function is the tree-sitter language function of a + // grammar `syntax-pack bundle` compiled from pinned, hashed sources + // and linked into this binary. + let language = unsafe { tree_sitter_language::LanguageFn::from_raw(bundled.language) }; + let grammar = Self::compile( + language, + bundled.abi, + bundled.highlights, + bundled.injections, + )?; + Ok(Self { + name: Some(Arc::from(bundled.name)), + locals: bundled.locals.map(Arc::from), + semantic: bundled.semantic.map(Arc::from), + tags: bundled.tags.map(Arc::from), + // Identifies the grammar build for tag caches, as a pack's + // library hash does. + library: Some(Arc::from(bundled.source)), + ..grammar + }) + } + pub(crate) fn name(&self) -> Option<&str> { self.name.as_deref() } @@ -112,32 +148,18 @@ impl Grammar { self.tags.as_ref() } - pub(crate) fn compile( - library_path: &Path, - symbol: &str, - expected_abi: Option, + /// Checks `language`'s ABI against the one its pack or bundle + /// declares and compiles its highlight and injection queries. + fn compile( + language: tree_sitter_language::LanguageFn, + expected_abi: u32, highlights: &str, injections: Option<&str>, ) -> Result { - // SAFETY: loading a library runs its initializers, and calling the - // symbol runs its code. The pack passed its SHA-256 check against a - // manifest that is either signed (downloaded packs) or in a - // directory the app opted into (local packs); see the threat model - // in the crate docs. - let language = unsafe { - let library = libloading::Library::new(library_path) - .map_err(|e| PackError::Library(e.to_string()))?; - let library: &'static libloading::Library = Box::leak(Box::new(library)); - let mut name = symbol.as_bytes().to_vec(); - name.push(0); - let symbol = library - .get:: *const ()>(&name) - .map_err(|e| PackError::Library(e.to_string()))?; - ts::Language::new(tree_sitter_language::LanguageFn::from_raw(*symbol)) - }; - // The manifest's ABI was checked; this checks the library agrees. + let language = ts::Language::new(language); + // The declared ABI was checked; this checks the grammar agrees. let abi = language.abi_version() as u32; - if expected_abi.is_some_and(|expected| expected != abi) || !supported_abi().contains(&abi) { + if expected_abi != abi || !supported_abi().contains(&abi) { let range = supported_abi(); return Err(PackError::Abi { abi, diff --git a/crates/diffy-syntax/src/lib.rs b/crates/diffy-syntax/src/lib.rs index 9d75798c..c25f10db 100644 --- a/crates/diffy-syntax/src/lib.rs +++ b/crates/diffy-syntax/src/lib.rs @@ -1,12 +1,15 @@ //! Tree-sitter syntax highlighting for code blocks, with grammars loaded at //! runtime. //! -//! No grammar is compiled in. Each language arrives as a pack, a shared -//! library exporting the tree-sitter language function plus its queries -//! and a manifest (see [`pack`]), from a [`GrammarStore`] the app +//! No grammar is compiled in by default. Each language arrives as a pack, +//! a shared library exporting the tree-sitter language function plus its +//! queries and a manifest (see [`pack`]), from a [`GrammarStore`] the app //! configures: local pack directories it trusts, and with the `download` //! feature a signed index it downloads missing packs from. Until a -//! language's grammar is available its code renders plain. +//! language's grammar is available its code renders plain. The exception +//! is the `bundled` feature, for builds that cannot load packs: it links +//! the grammars `syntax-pack bundle` compiled (see [`bundled`] and +//! PACKS.md), used after [`StoreConfig::bundled`]. //! //! [`highlight`] runs synchronously; [`HighlightWorker`] runs it on a //! background thread and drops requests superseded by a newer generation @@ -24,9 +27,9 @@ //! an embedded grammar downloads, its region keeps the host's colors, and //! the worker sends a newer revision of the result when it arrives. //! -//! Features: `engine` (the tree-sitter runtime and local packs) and -//! `download` (fetching packs). Without `engine` every lookup is plain and -//! the crate has no dependencies. +//! Features: `engine` (the tree-sitter runtime and local packs), +//! `download` (fetching packs), and `bundled` (grammars linked in). Without +//! `engine` every lookup is plain and the crate has no dependencies. //! //! # Threat model //! @@ -56,7 +59,12 @@ //! - Grammars parse untrusted text (chat output, diffs). A parser crash //! takes the process down; a panic in the highlighter is caught by the //! worker. Packs come from pinned, hashed grammar sources built in CI. +//! - Bundled grammars are part of the binary, built from the same pinned, +//! hashed sources, so they are as trusted as the rest of it; nothing is +//! loaded or hashed at run time. +#[cfg(feature = "bundled")] +pub mod bundled; #[cfg(feature = "download")] mod download; #[cfg(feature = "engine")] diff --git a/crates/diffy-syntax/src/store.rs b/crates/diffy-syntax/src/store.rs index 4c92ef34..2484cfd8 100644 --- a/crates/diffy-syntax/src/store.rs +++ b/crates/diffy-syntax/src/store.rs @@ -79,12 +79,14 @@ pub enum LanguageStatus { /// The grammars an app can highlight with, shared by every highlighter. /// Cloning is cheap and clones share state. /// -/// Packs come from, in order: +/// Grammars come from, in order: /// /// 1. local pack roots the app opts into ([`StoreConfig::local_packs`]), /// such as packs bundled with the app or built by `syntax-pack` in /// development; -/// 2. with the `download` feature, the cache of a signed index the app +/// 2. with the `bundled` feature, the grammars compiled into the binary +/// ([`StoreConfig::bundled`]); +/// 3. with the `download` feature, the cache of a signed index the app /// configures ([`StoreConfig::downloads`]), downloading a missing pack /// on a background thread the first time its language is asked for. /// @@ -536,6 +538,8 @@ impl Outcome { #[derive(Debug, Clone, Default)] pub struct StoreConfig { local_packs: Vec, + #[cfg(feature = "bundled")] + bundled: bool, /// Languages whose local packs stay pending until /// [`GrammarStore::release_deferred`], as if they were downloading. deferred: Vec, @@ -569,6 +573,14 @@ impl StoreConfig { self } + /// Uses the grammars compiled into the binary ([`crate::bundled`]), + /// after the local pack roots and before any download. + #[cfg(feature = "bundled")] + pub fn bundled(mut self) -> Self { + self.bundled = true; + self + } + /// Downloads missing packs from a signed index. #[cfg(feature = "download")] pub fn downloads(mut self, downloads: crate::download::Downloads) -> Self { @@ -607,9 +619,9 @@ pub(crate) struct State { pub(crate) tags: HashMap, /// Tag to pack directory over the local roots, scanned on first lookup. local: Option)>>, - /// Grammars by pack directory, so a language's aliases share one load. - /// `None` records a pack that failed to load. - pub(crate) loaded: HashMap>>, + /// Grammars by where they came from, so a language's aliases share one + /// load. `None` records a grammar that failed to load. + pub(crate) loaded: HashMap>>, /// [`StoreConfig::defer`]red packs have been released. released: bool, /// Tags left pending by [`StoreConfig::defer`]. @@ -618,6 +630,38 @@ pub(crate) struct State { pub(crate) remote: crate::download::Remote, } +/// Where a grammar comes from. +#[cfg(feature = "engine")] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) enum Origin { + /// A pack directory, local or in the download cache. + Pack(PathBuf), + /// A grammar compiled in, by name. + #[cfg_attr(not(feature = "bundled"), allow(dead_code))] + Bundled(&'static str), +} + +/// A language the catalog lists. +#[cfg(feature = "engine")] +struct Known { + origin: Origin, + language: String, + tags: Vec, + installed: bool, +} + +#[cfg(feature = "engine")] +impl Known { + fn pack(dir: PathBuf, manifest: &PackManifest, installed: bool) -> Self { + Self { + origin: Origin::Pack(dir), + language: manifest.language.clone(), + tags: manifest.tags().map(str::to_owned).collect(), + installed, + } + } +} + #[cfg(feature = "engine")] pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { // A panic while holding the lock leaves maps that are still valid. @@ -653,6 +697,12 @@ impl Inner { if let Some((_, manifest)) = local.get(tag) { return manifest.tags.is_some(); } + #[cfg(feature = "bundled")] + if self.config.bundled + && let Some(bundled) = crate::bundled::find(tag) + { + return bundled.tags.is_some(); + } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { return crate::download::ships_tags(downloads, &mut state, tag); @@ -671,6 +721,14 @@ impl Inner { } return load(state, &dir, &manifest); } + #[cfg(feature = "bundled")] + if self.config.bundled + && let Some(bundled) = crate::bundled::find(tag) + { + return load_with(state, Origin::Bundled(bundled.name), || { + Grammar::bundled(bundled) + }); + } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { return crate::download::resolve(self, downloads, state, tag); @@ -684,58 +742,71 @@ impl Inner { .local .get_or_insert_with(|| scan_local(&self.config.local_packs)) .clone(); - let mut packs: Vec<(PathBuf, Arc, bool)> = Vec::new(); + let mut known: Vec = Vec::new(); let mut seen = std::collections::HashSet::new(); let mut local_packs: Vec<_> = local.values().cloned().collect(); local_packs.sort_by(|a, b| a.0.cmp(&b.0)); for (dir, manifest) in local_packs { if seen.insert(manifest.language.clone()) { - packs.push((dir, manifest, true)); + known.push(Known::pack(dir, &manifest, true)); + } + } + #[cfg(feature = "bundled")] + if self.config.bundled { + for bundled in crate::bundled::GRAMMARS { + if seen.insert(bundled.name.to_owned()) { + known.push(Known { + origin: Origin::Bundled(bundled.name), + language: bundled.name.to_owned(), + tags: bundled.tags().map(str::to_owned).collect(), + installed: true, + }); + } } } #[cfg(feature = "download")] if let Some(downloads) = &self.config.downloads { for (dir, manifest, installed) in crate::download::packs(downloads, &mut state) { if seen.insert(manifest.language.clone()) { - packs.push((dir, manifest, installed)); + known.push(Known::pack(dir, &manifest, installed)); } } } let mut catalog = Catalog { - languages: packs.len(), + languages: known.len(), ..Catalog::default() }; - for (dir, manifest, installed) in &packs { - for tag in manifest.tags() { + for entry in &known { + for tag in &entry.tags { catalog .tags .entry(tag.to_ascii_lowercase()) - .or_insert_with(|| manifest.language.clone()); + .or_insert_with(|| entry.language.clone()); } - let pending = manifest.tags().any(|tag| { + let pending = entry.tags.iter().any(|tag| { matches!( state.tags.get(&tag.to_ascii_lowercase()), Some(Tag::Pending) ) }); #[cfg(feature = "download")] - let failure = state.remote.failure(&manifest.language); + let failure = state.remote.failure(&entry.language); #[cfg(not(feature = "download"))] let failure: Option = None; - let grammar_state = if matches!(state.loaded.get(dir), Some(Some(_))) { + let grammar_state = if matches!(state.loaded.get(&entry.origin), Some(Some(_))) { Some(GrammarState::Ready) } else if let Some(reason) = failure { Some(GrammarState::Failed(reason)) } else if pending { Some(GrammarState::Downloading) - } else if *installed { + } else if entry.installed { Some(GrammarState::Installed) } else { None }; if let Some(grammar_state) = grammar_state { catalog.statuses.push(GrammarStatus { - language: manifest.language.clone(), + language: entry.language.clone(), state: grammar_state, }); } @@ -757,22 +828,24 @@ impl Inner { /// Loads (once per directory) the pack in `dir`. #[cfg(feature = "engine")] pub(crate) fn load(state: &mut State, dir: &Path, manifest: &PackManifest) -> Tag { - load_with(state, dir, || Grammar::load(dir, manifest)) + load_with(state, Origin::Pack(dir.to_path_buf()), || { + Grammar::load(dir, manifest) + }) } #[cfg(feature = "engine")] pub(crate) fn load_with( state: &mut State, - dir: &Path, + origin: Origin, load: impl FnOnce() -> Result, ) -> Tag { let grammar = state .loaded - .entry(dir.to_path_buf()) - .or_insert_with(|| match load() { + .entry(origin) + .or_insert_with_key(|origin| match load() { Ok(grammar) => Some(Arc::new(grammar)), Err(error) => { - tracing::warn!(pack = %dir.display(), %error, "syntax pack failed to load"); + tracing::warn!(?origin, %error, "grammar failed to load"); None } }); diff --git a/crates/diffy-syntax/src/tests.rs b/crates/diffy-syntax/src/tests.rs index bb1e71c1..d6d4cbd5 100644 --- a/crates/diffy-syntax/src/tests.rs +++ b/crates/diffy-syntax/src/tests.rs @@ -1535,3 +1535,212 @@ mod navigate { assert_eq!(lookup.import, Some(pos(0, 26))); } } + +#[cfg(feature = "bundled")] +mod bundled { + use super::*; + use HighlightKind::*; + + /// Each default bundled language: a tag that should reach it, its + /// sample (a file under tools/syntax-pack/samples, or inline source), + /// and text the sample must color with a kind, as `syntax-pack check` + /// expects of its captures. + type Case = ( + &'static str, + &'static str, + &'static str, + &'static [(&'static str, HighlightKind)], + ); + const CASES: &[Case] = &[ + ( + "c", + "h", + "c.c", + &[ + ("static", Keyword), + ("greet", Function), + ("/* Prints", Comment), + ], + ), + ( + "cpp", + "cc", + "cpp.cpp", + &[("namespace", Keyword), ("\"hello, \"", String)], + ), + ( + "css", + "css", + "css.css", + &[("@media", Keyword), ("color", Property), ("4px", Number)], + ), + ( + "go", + "golang", + "go.go", + &[("func", Keyword), ("Greeter", Type), ("42", Number)], + ), + ( + "html", + "htm", + "html.html", + &[("html", Tag), ("lang", Attribute)], + ), + ( + "javascript", + "jsx", + "javascript.js", + &[("function", Keyword), ("p", Tag)], + ), + ( + "json", + "jsonc", + "json.json", + &[("\"quark\"", String), ("1", Number), ("true", Constant)], + ), + ( + "kotlin", + "kts", + "fun main() {\n val greeting = \"hi\"\n}\n", + &[("fun", Keyword), ("\"hi\"", String)], + ), + // Its inline spans come from markdown_inline and the fence from + // rust, both bundled, through its injection query. + ( + "markdown", + "md", + "markdown.md", + &[("Hello", Keyword), ("bold", Type), ("fn", Keyword)], + ), + ( + "markdown_inline", + "markdown_inline", + "markdown_inline.md", + &[("emphasis", Attribute), ("code", String)], + ), + ( + "python", + "py", + "python.py", + &[("class", Keyword), ("greet", Function), ("42", Number)], + ), + ( + "rust", + "rs", + "rust.rs", + &[("fn", Keyword), ("Greeter", Type), ("// Says", Comment)], + ), + ( + "swift", + "swift", + "func greet(name: String) -> String {\n return \"hi\"\n}\n", + &[("func", Keyword), ("\"hi\"", String)], + ), + ( + "toml", + "toml", + "toml.toml", + &[ + ("# Package", Comment), + ("\"quark\"", String), + ("true", Constant), + ], + ), + ( + "tsx", + "tsx", + "tsx.tsx", + &[("type", Keyword), ("Props", Type), ("div", Tag)], + ), + ( + "typescript", + "mts", + "typescript.ts", + &[("interface", Keyword), ("Greeter", Type)], + ), + ( + "yaml", + "yml", + "yaml.yaml", + &[ + ("name", Property), + ("true", Constant), + ("# Workflow", Comment), + ], + ), + ]; + + fn sample(sample: &str) -> std::string::String { + if sample.contains('\n') { + return sample.to_owned(); + } + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tools/syntax-pack/samples") + .join(sample); + std::fs::read_to_string(&path).unwrap() + } + + // Catches a bundled grammar that is not linked, not reached by its + // tags, whose queries do not compile, or whose injections do not reach + // the other bundled grammars. Needs a bundle with the default set + // (`syntax-pack bundle`, then DIFFY_SYNTAX_BUNDLE). + #[test] + fn bundled_grammars_resolve_by_tag_and_highlight_their_samples() { + if crate::bundled::languages().len() == 0 { + eprintln!("skipping: built without DIFFY_SYNTAX_BUNDLE"); + return; + } + let store = GrammarStore::new(StoreConfig::new().bundled()); + + for &(name, tag, file, expected) in CASES { + assert!( + crate::bundled::languages().any(|l| l == name), + "{name} is not bundled" + ); + assert_eq!(store.status(&language(tag)), LanguageStatus::Ready, "{tag}"); + let source = sample(file); + let spans = highlight(&store, &language(tag), &source); + for &(text, kind) in expected { + let found = source.match_indices(text).any(|(at, _)| { + spans + .iter() + .any(|span| span.kind == kind && span.range().contains(&at)) + }); + assert!(found, "{name}: {text:?} is not {kind:?}"); + } + } + let rust = LanguageId::from_path("src/lib.rs").unwrap(); + assert_eq!(store.status(&rust), LanguageStatus::Ready); + } + + // Catches bundled grammars missing from the catalog, or a store that + // did not opt in seeing them. + #[test] + fn the_catalog_lists_bundled_grammars_only_when_the_store_opts_in() { + if crate::bundled::languages().len() == 0 { + return; + } + let store = GrammarStore::new(StoreConfig::new().bundled()); + store.status(&language("rs")); + + let catalog = store.catalog(); + + assert_eq!(catalog.languages, crate::bundled::languages().len()); + assert_eq!( + catalog.tags.get("rs").map(std::string::String::as_str), + Some("rust") + ); + let state = |name: &str| { + catalog + .statuses + .iter() + .find(|status| status.language == name) + .map(|status| status.state.clone()) + }; + assert_eq!(state("rust"), Some(GrammarState::Ready)); + assert_eq!(state("go"), Some(GrammarState::Installed)); + let plain = GrammarStore::new(StoreConfig::new()); + assert_eq!(plain.status(&language("rs")), LanguageStatus::Unavailable); + assert_eq!(plain.catalog().languages, 0); + } +} diff --git a/tools/syntax-pack/README.md b/tools/syntax-pack/README.md index 6773a13f..89d034c1 100644 --- a/tools/syntax-pack/README.md +++ b/tools/syntax-pack/README.md @@ -20,6 +20,33 @@ subcommand. to one language and is lowercase, and that each expected injection is a pinned language's tag. +## Bundles + +`bundle` compiles grammars into static libraries for diffy-syntax's +`bundled` feature; usage is in the +[guide](../../crates/diffy-syntax/PACKS.md#bundled-grammars). + +- Each language is first built and checked as a pack for the host, from the + same fetched and hash-checked sources, so its queries are adapted even + for another target. +- Static libraries are compiled with `-fvisibility=hidden` and PIC, and + with the dynamic CRT (`/MD`) on MSVC, as Rust uses by default. +- All libraries are then linked into one throwaway shared library for the + target, so two grammars defining one global symbol fail the bundle. +- C++ scanners are refused: every consumer would need the C++ runtime. +- `THIRD_PARTY_NOTICES.md` copies the license files at the top of each + grammar repository (required) and any query repository (the + nvim-treesitter-queries-* ones have none). +- A cross-target bundle needs the target's C compiler and archiver as + `build` does (`CC_`, `AR_`), plus the host's. + +```sh +# Per target, on any runner with that target's C toolchain: +cargo run -p syntax-pack -- bundle --out "$PWD/bundle" --target aarch64-apple-darwin +# Then, for the consumer: +DIFFY_SYNTAX_BUNDLE="$PWD/bundle" cargo build --release --target aarch64-apple-darwin --features diffy-syntax/bundled +``` + ## Languages - [grammars.toml](grammars.toml): the hand-pinned core languages, each with diff --git a/tools/syntax-pack/src/main.rs b/tools/syntax-pack/src/main.rs index 13ddafe4..d268d221 100644 --- a/tools/syntax-pack/src/main.rs +++ b/tools/syntax-pack/src/main.rs @@ -11,6 +11,13 @@ //! does. A registry language that fails is left out and recorded in //! //skipped.json; a grammars.toml language fails the build. //! Grammars marked `generate` need tree-sitter CLI 0.26.8 ($TREE_SITTER). +//! syntax-pack bundle [LANGUAGE...] [--target TRIPLE] [--out DIR] [--sources DIR] [--work DIR] [--jobs N] +//! Compile LANGUAGEs (default: BUNDLE_DEFAULT) from their pinned, +//! hashed sources into static libraries for TRIPLE, for diffy-syntax's +//! `bundled` feature (DIFFY_SYNTAX_BUNDLE=DIR). Each is first built and +//! checked as a pack for this host, which adapts its queries. Writes +//! //: lib/, queries//, bundle.tsv, and +//! THIRD_PARTY_NOTICES.md. --out defaults to /target/syntax-bundle. //! syntax-pack registry --registry FILE [--extensions FILE] [--jobs N] //! Pin every grammar in nvim-treesitter's registry.json that is not in //! grammars.toml, build and check each for this host, and rewrite @@ -68,6 +75,27 @@ const HOST: &str = env!("SYNTAX_PACK_HOST"); const KEY_VAR: &str = "DIFFY_SYNTAX_SIGNING_KEY"; const TREE_SITTER_CLI: &str = "0.26.8"; const TREE_SITTER_VAR: &str = "TREE_SITTER"; +/// What `bundle` compiles in when no languages are named. `markdown_inline` +/// is there because markdown's injection query names it. +const BUNDLE_DEFAULT: &[&str] = &[ + "c", + "cpp", + "css", + "go", + "html", + "javascript", + "json", + "kotlin", + "markdown", + "markdown_inline", + "python", + "rust", + "swift", + "toml", + "tsx", + "typescript", + "yaml", +]; fn main() -> ExitCode { let args: Vec = std::env::args().skip(1).collect(); @@ -80,6 +108,7 @@ fn main() -> ExitCode { } } +#[derive(Clone)] struct Options { languages: Vec, target: String, @@ -155,7 +184,7 @@ fn run(args: &[String]) -> Result<(), String> { let options = options()?; let mut skipped = BTreeMap::new(); let results = registry::par_map(&options.languages, options.jobs, |language| { - build(&lock, language, &options).and_then(|()| { + build(&lock, language, &options).and_then(|_| { if options.target == HOST { check_language(&lock, language, &options) } else { @@ -177,6 +206,16 @@ fn run(args: &[String]) -> Result<(), String> { } write_skipped(&options, &skipped) } + Some("bundle") => { + let mut options = parse_options(&args[1..])?; + if options.languages.is_empty() { + options.languages = BUNDLE_DEFAULT.iter().map(|l| (*l).to_owned()).collect(); + } + if !args.iter().any(|arg| arg == "--out") { + options.out = workspace().join("target/syntax-bundle"); + } + bundle(&lock, &options) + } Some("check") => check(&lock, &options()?), Some("index") => index(&lock, &options()?), Some("verify") => verify(&lock, &options()?), @@ -201,7 +240,7 @@ fn run(args: &[String]) -> Result<(), String> { Ok(()) } _ => Err( - "usage: syntax-pack build|check|index|verify|remote-check|registry|public-key|list \ + "usage: syntax-pack build|bundle|check|index|verify|remote-check|registry|public-key|list \ (see src/main.rs)" .to_owned(), ), @@ -419,7 +458,9 @@ fn inputs( Ok(inputs) } -fn build(lock: &Lock, language: &str, options: &Options) -> Result<(), String> { +/// Builds `language`'s pack; returns the directory of the C sources it +/// compiled (generated ones for a `generate` grammar). +fn build(lock: &Lock, language: &str, options: &Options) -> Result { let grammar = lock .get(language) .ok_or_else(|| format!("{language} is not pinned"))?; @@ -560,7 +601,7 @@ fn build(lock: &Lock, language: &str, options: &Options) -> Result<(), String> { out.join(&library_name).display(), manifest.library.size ); - Ok(()) + Ok(src) } fn pack_file(dir: &Path, name: &str) -> Result { @@ -615,6 +656,22 @@ fn copy_dir(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } +/// A C build of the sources in `src` for `target`, objects in `objects`. +fn c_build(src: &Path, target: &str, objects: &Path) -> cc::Build { + let mut build = cc::Build::new(); + build + .cargo_metadata(false) + .cargo_warnings(false) + .warnings(false) + .target(target) + .host(HOST) + .opt_level(2) + .debug(false) + .out_dir(objects) + .include(src); + build +} + /// Compiles parser.c and its scanner in `src` into the shared library /// `output` for `target`. fn compile( @@ -626,18 +683,8 @@ fn compile( ) -> Result<(), String> { let _ = fs::remove_dir_all(objects); fs::create_dir_all(objects).map_err(|e| e.to_string())?; - let mut build = cc::Build::new(); - build - .cargo_metadata(false) - .cargo_warnings(false) - .warnings(false) - .target(target) - .host(HOST) - .opt_level(2) - .debug(false) - .out_dir(objects) - .include(src) - .file(src.join("parser.c")); + let mut build = c_build(src, target, objects); + build.file(src.join("parser.c")); if scanner == Scanner::C { build.file(src.join("scanner.c")); } @@ -657,32 +704,42 @@ fn compile( .try_compile_intermediates() .map_err(|e| e.to_string())?; if scanner == Scanner::Cc { - let mut cxx = cc::Build::new(); - cxx.cargo_metadata(false) - .cargo_warnings(false) - .warnings(false) - .cpp(true) - .target(target) - .host(HOST) - .opt_level(2) - .debug(false) - .out_dir(objects) - .include(src) - .file(src.join("scanner.cc")); + let mut cxx = c_build(src, target, objects); + cxx.cpp(true).file(src.join("scanner.cc")); if !compiler.is_like_msvc() { cxx.flag("-fvisibility=hidden"); } objs.extend(cxx.try_compile_intermediates().map_err(|e| e.to_string())?); } - let mut link = if compiler.is_like_msvc() { + let mut link = shared_link(&compiler, target, objects, output)?; + link.args(&objs); + if scanner == Scanner::Cc && !compiler.is_like_msvc() { + link.arg(if target.contains("apple") { + "-lc++" + } else { + "-lstdc++" + }); + } + run_link(link, output) +} + +/// The command that links a shared library `output` for `target`; the +/// caller adds the inputs. +fn shared_link( + compiler: &cc::Tool, + target: &str, + objects: &Path, + output: &Path, +) -> Result { + if compiler.is_like_msvc() { let mut link = cc::windows_registry::find_tool(target, "link.exe") .ok_or("link.exe not found; run from a Visual Studio developer shell")? .to_command(); link.args(["/NOLOGO", "/DLL"]) .arg(format!("/OUT:{}", output.display())) .arg(format!("/IMPLIB:{}", objects.join("parser.lib").display())); - link + Ok(link) } else { let mut link = compiler.to_command(); if target.contains("apple") { @@ -691,16 +748,11 @@ fn compile( link.args(["-shared", "-s"]); } link.arg("-o").arg(output); - link - }; - link.args(&objs); - if scanner == Scanner::Cc && !compiler.is_like_msvc() { - link.arg(if target.contains("apple") { - "-lc++" - } else { - "-lstdc++" - }); + Ok(link) } +} + +fn run_link(mut link: Command, output: &Path) -> Result<(), String> { let status = link.status().map_err(|e| format!("linker: {e}"))?; if !status.success() { return Err(format!("linking {} failed: {status}", output.display())); @@ -708,6 +760,327 @@ fn compile( Ok(()) } +// ---- Bundle ------------------------------------------------------------- + +/// The static library name of a bundled language. +fn bundle_library(language: &str) -> String { + let name: String = language + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + format!("diffy_grammar_{name}") +} + +/// The file name `cc` gives the static library `name` for `target`, and +/// rustc looks for. +fn static_library_file(target: &str, name: &str) -> String { + if target.contains("msvc") { + format!("{name}.lib") + } else { + format!("lib{name}.a") + } +} + +fn bundle(lock: &Lock, options: &Options) -> Result<(), String> { + let target = &options.target; + let mut libraries = BTreeMap::new(); + for language in &options.languages { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + // Linking one would need the C++ runtime in every consumer. + if grammar.scanner == Scanner::Cc { + return Err(format!("{language}: C++ scanners cannot be bundled")); + } + if let Some(other) = libraries.insert(bundle_library(language), language) { + return Err(format!("{other} and {language} share a library name")); + } + } + let out = options.out.join(target); + let _ = fs::remove_dir_all(&out); + let lib_dir = out.join("lib"); + fs::create_dir_all(&lib_dir).map_err(|e| format!("{}: {e}", lib_dir.display()))?; + // Packs for this host, whatever the target: building one adapts its + // queries to tree-sitter and checks them, and its sample, against the + // grammar the bundle compiles from the same sources. + let host = Options { + target: HOST.to_owned(), + out: options.work.join("bundle-packs"), + ..options.clone() + }; + let results = registry::par_map(&options.languages, options.jobs, |language| { + let src = build(lock, language, &host)?; + check_language(lock, language, &host)?; + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + let objects = options + .work + .join("bundle-objects") + .join(target) + .join(language); + let library = bundle_library(language); + eprintln!("compiling {language} into {library} for {target}"); + compile_static(&src, grammar.scanner, target, &objects, &library, &lib_dir) + .map_err(|e| format!("{language}: {e}")) + }); + for result in results { + result?; + } + + let mut table = String::from( + "# diffy-syntax bundle 1\n\ + # language\tsymbol\tabi\tlibrary\taliases\textensions\tsource sha256\n", + ); + let mut symbols = BTreeSet::new(); + for language in &options.languages { + let dir = host.out.join(HOST).join(language); + let manifest = pack::read_manifest(&dir).map_err(|e| format!("{language}: {e}"))?; + if !symbols.insert(manifest.symbol.clone()) { + return Err(format!("{language}: symbol {} is taken", manifest.symbol)); + } + let queries = out.join("queries").join(language); + fs::create_dir_all(&queries).map_err(|e| e.to_string())?; + for (name, file) in [ + ("highlights", Some(&manifest.highlights)), + ("injections", manifest.injections.as_ref()), + ("locals", manifest.locals.as_ref()), + ("semantic", manifest.semantic.as_ref()), + ("tags", manifest.tags.as_ref()), + ] { + if let Some(file) = file { + let to = queries.join(format!("{name}.scm")); + fs::copy(dir.join(&file.path), &to) + .map_err(|e| format!("{}: {e}", to.display()))?; + } + } + let list = |tags: &[String]| match tags.join(",") { + joined if joined.is_empty() => "-".to_owned(), + joined => joined, + }; + let source = manifest + .source + .as_ref() + .ok_or_else(|| format!("{language}: the pack names no source"))?; + table.push_str(&format!( + "{language}\t{}\t{}\t{}\t{}\t{}\t{}\n", + manifest.symbol, + manifest.abi, + bundle_library(language), + list(&manifest.aliases), + list(&manifest.extensions), + source.sha256, + )); + } + probe_link(options, &symbols)?; + let notices = out.join("THIRD_PARTY_NOTICES.md"); + fs::write(¬ices, third_party_notices(lock, options)?) + .map_err(|e| format!("{}: {e}", notices.display()))?; + // Written last: diffy-syntax refuses a bundle without it. + fs::write(out.join("bundle.tsv"), table).map_err(|e| e.to_string())?; + for language in &options.languages { + let path = lib_dir.join(static_library_file(target, &bundle_library(language))); + let size = fs::metadata(&path).map_err(|e| e.to_string())?.len(); + println!("{} ({size} bytes)", path.display()); + } + println!("{}", out.display()); + Ok(()) +} + +/// Compiles parser.c and its C scanner in `src` into the static library +/// `library` for `target`, copied into `lib_dir`. +fn compile_static( + src: &Path, + scanner: Scanner, + target: &str, + objects: &Path, + library: &str, + lib_dir: &Path, +) -> Result<(), String> { + let _ = fs::remove_dir_all(objects); + fs::create_dir_all(objects).map_err(|e| format!("{}: {e}", objects.display()))?; + let mut build = c_build(src, target, objects); + // Position-independent so a shared library (a Node addon) can link it; + // the dynamic CRT on MSVC, as Rust uses by default. + build.pic(true).static_crt(false).file(src.join("parser.c")); + if scanner == Scanner::C { + build.file(src.join("scanner.c")); + } + let compiler = build.try_get_compiler().map_err(|e| e.to_string())?; + if !compiler.is_like_msvc() { + build.flag("-std=c11").flag("-fvisibility=hidden"); + } + build.try_compile(library).map_err(|e| e.to_string())?; + let file = static_library_file(target, library); + fs::copy(objects.join(&file), lib_dir.join(&file)).map_err(|e| format!("{file}: {e}"))?; + Ok(()) +} + +/// Links every library of the bundle into one shared library, as a +/// consumer's build would link them into one binary: two grammars defining +/// the same global symbol fail here, for this target, instead of there. +fn probe_link(options: &Options, symbols: &BTreeSet) -> Result<(), String> { + let target = &options.target; + let objects = options + .work + .join("bundle-objects") + .join(target) + .join("probe"); + let _ = fs::remove_dir_all(&objects); + fs::create_dir_all(&objects).map_err(|e| e.to_string())?; + let mut probe = String::new(); + for symbol in symbols { + probe.push_str(&format!("const void *{symbol}(void);\n")); + } + probe.push_str("const void *(*const diffy_bundle_probe[])(void) = {\n"); + for symbol in symbols { + probe.push_str(&format!(" {symbol},\n")); + } + probe.push_str("};\n"); + let source = objects.join("probe.c"); + fs::write(&source, probe).map_err(|e| e.to_string())?; + let mut build = c_build(&objects, target, &objects); + build.pic(true).static_crt(false).file(&source); + let compiler = build.try_get_compiler().map_err(|e| e.to_string())?; + let objs = build + .try_compile_intermediates() + .map_err(|e| format!("probe: {e}"))?; + let output = objects.join(match pack::library_suffix(target) { + ".dll" => "probe.dll".to_owned(), + suffix => format!("libprobe{suffix}"), + }); + let mut link = shared_link(&compiler, target, &objects, &output)?; + link.args(&objs); + let lib_dir = options.out.join(target).join("lib"); + for language in &options.languages { + link.arg(lib_dir.join(static_library_file(target, &bundle_library(language)))); + } + run_link(link, &output).map_err(|e| format!("the bundle does not link as one: {e}")) +} + +/// Each bundled grammar's license, the repositories and commits its +/// sources and queries come from, and the license files found in them. +fn third_party_notices(lock: &Lock, options: &Options) -> Result { + let mut text = String::from( + "# Third-party notices\n\n\ + These tree-sitter grammars and queries are compiled into this software.\n", + ); + for language in &options.languages { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + text.push_str(&format!( + "\n## {language}\n\nLicense: {}\n\n", + grammar.license.as_deref().unwrap_or("unknown") + )); + let sources = source_repositories(lock, language, options)?; + for (index, (repo, rev, root)) in sources.into_iter().enumerate() { + text.push_str(&format!("Source: {repo} at {rev}\n\n")); + let licenses = license_files(&root)?; + // The grammar's repository holds the C code compiled in; query + // repositories (nvim-treesitter-queries-*) carry no license file. + if licenses.is_empty() && index == 0 { + return Err(format!( + "{language}: {repo} at {rev} has no license file to ship" + )); + } + if licenses.is_empty() { + text.push_str("This repository has no license file.\n\n"); + } + for path in licenses { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let license = read_text(&path)?; + text.push_str(&format!( + "{name}:\n\n```text\n{}\n```\n\n", + license.trim_end() + )); + } + } + } + Ok(text) +} + +/// The repository, commit, and checkout of every source a language's +/// build reads: its grammar, its query repository, and other entries' +/// files it names (`@/`, `@:q/`). +fn source_repositories( + lock: &Lock, + language: &str, + options: &Options, +) -> Result, String> { + let grammar = lock + .get(language) + .ok_or_else(|| format!("{language} is not pinned"))?; + let mut found = vec![( + grammar.repo.clone(), + grammar.rev.clone(), + checkout(lock, language, options)?, + )]; + let names = grammar + .include + .iter() + .chain(&grammar.highlights) + .chain(&grammar.injections) + .chain(&grammar.locals) + .chain(&grammar.tags); + for name in names { + let (owner, queries) = if name.starts_with("q:") { + (language, true) + } else if let Some((other, _)) = name.strip_prefix('@').and_then(|r| r.split_once('/')) { + match other.strip_suffix(":q") { + Some(other) => (other, true), + None => (other, false), + } + } else { + continue; + }; + let (source, root) = if queries { + let source = lock + .queries_of(owner) + .ok_or_else(|| format!("{owner} has no pinned queries"))?; + (source, queries_checkout(lock, owner, options)?) + } else { + let other = lock + .get(owner) + .ok_or_else(|| format!("{owner} is not pinned"))?; + let source = Source { + repo: other.repo.clone(), + rev: other.rev.clone(), + }; + (source, checkout(lock, owner, options)?) + }; + if !found + .iter() + .any(|(repo, rev, _)| *repo == source.repo && *rev == source.rev) + { + found.push((source.repo, source.rev, root)); + } + } + Ok(found) +} + +/// The license files at the top of a checkout, by name. +fn license_files(root: &Path) -> Result, String> { + let entries = fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))?; + let mut found: Vec = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + let name = path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_ascii_uppercase(); + path.is_file() + && ["LICENSE", "LICENCE", "COPYING"] + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .collect(); + found.sort(); + Ok(found) +} + // ---- Check -------------------------------------------------------------- fn check(lock: &Lock, options: &Options) -> Result<(), String> { From 5c0517dcad7abfc8fda327ada7199d7ffffbbf68 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:19:10 +0000 Subject: [PATCH 04/16] fix(syntax-pack): credit nvim-treesitter's Apache-2.0 queries in bundle notices --- tools/syntax-pack/src/main.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tools/syntax-pack/src/main.rs b/tools/syntax-pack/src/main.rs index d268d221..c7e25748 100644 --- a/tools/syntax-pack/src/main.rs +++ b/tools/syntax-pack/src/main.rs @@ -965,6 +965,7 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result "# Third-party notices\n\n\ These tree-sitter grammars and queries are compiled into this software.\n", ); + let mut nvim_queries = false; for language in &options.languages { let grammar = lock .get(language) @@ -984,7 +985,15 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result "{language}: {repo} at {rev} has no license file to ship" )); } - if licenses.is_empty() { + if licenses.is_empty() && repo.contains("nvim-treesitter-queries-") { + // These repositories split out nvim-treesitter's queries, + // which it licenses Apache-2.0; the text follows the list. + nvim_queries = true; + text.push_str( + "Queries from nvim-treesitter, licensed Apache-2.0 \ + (text at the end of this file).\n\n", + ); + } else if licenses.is_empty() { text.push_str("This repository has no license file.\n\n"); } for path in licenses { @@ -997,9 +1006,20 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result } } } + if nvim_queries { + text.push_str(&format!( + "\n## nvim-treesitter queries\n\n\ + Source: https://github.com/nvim-treesitter/nvim-treesitter\n\n\ + ```text\n{}\n```\n", + APACHE_2_0.trim_end() + )); + } Ok(text) } +/// The Apache License 2.0, verbatim. +const APACHE_2_0: &str = include_str!("../../../crates/diffy-syntax/LICENSE-APACHE"); + /// The repository, commit, and checkout of every source a language's /// build reads: its grammar, its query repository, and other entries' /// files it names (`@/`, `@:q/`). From 71086779d0bd22875f53f6d97446a2def4ca2c01 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:19:41 +0000 Subject: [PATCH 05/16] refactor(compare): move the semantic compare chain into diffy-compare diffy-compare now implements diff_file/diff_files: grammar lookup and rule compilation, injection embedding, the semantic diff, rows-to-document building, and the line diff (histogram + Git slider heuristics via gix-imara-diff, the same crate and version gix links). Documents are byte-for-byte what the app built before. New: per-file semantic budgets capped by a request deadline (files after it get line diffs with Fallback::RequestBudget) and cancellation between files (Error::Cancelled). The app keeps only the Git-facing parts in compare::semantic, uses FileInput in place of ChangedPath, reports progress and energy work through an Observer, and logs fallbacks from FileDiff. semantic-eval calls diffy-compare directly. --- AGENTS.md | 2 + Cargo.lock | 4 + Cargo.toml | 1 + crates/diffy-compare/Cargo.toml | 5 + crates/diffy-compare/src/document.rs | 369 +++++++++++ crates/diffy-compare/src/file.rs | 156 +++++ crates/diffy-compare/src/grammar.rs | 71 ++ crates/diffy-compare/src/lib.rs | 58 +- crates/diffy-compare/src/line.rs | 55 ++ crates/diffy-compare/src/run.rs | 112 ++++ crates/diffy-compare/src/tests.rs | 303 +++++++++ src/app/viewport/pipeline/state.rs | 2 +- src/core/compare/git.rs | 72 +- src/core/compare/semantic.rs | 948 +++------------------------ src/core/error.rs | 10 + src/core/vcs/jj/service.rs | 8 +- tools/semantic-eval/Cargo.toml | 1 + tools/semantic-eval/src/engine.rs | 18 +- 18 files changed, 1256 insertions(+), 939 deletions(-) create mode 100644 crates/diffy-compare/src/document.rs create mode 100644 crates/diffy-compare/src/file.rs create mode 100644 crates/diffy-compare/src/grammar.rs create mode 100644 crates/diffy-compare/src/line.rs create mode 100644 crates/diffy-compare/src/run.rs create mode 100644 crates/diffy-compare/src/tests.rs diff --git a/AGENTS.md b/AGENTS.md index 82ba3784..c3907b2d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,8 @@ and syntax code stays here. - `src/apprt/` — runtime and services split per stream; all blocking Git/fs/network/AI work runs on workers and reports through `transport.rs`. - `src/core/` — product logic: compare backends, VCS, review, syntax, themes. +- `crates/diffy-compare/` — headless chain from a changed file's bytes to a + `DiffDocument` (semantic diff, else line diff); no UI, Git or network deps. - `crates/diffy-diff/` — diff model; `crates/diffy-ui/` — diff view on Quark; `crates/diffy-syntax/` — tree-sitter packs (see its `PACKS.md`); `tools/syntax-pack/` — pack builder; `registry.toml` is generated by diff --git a/Cargo.lock b/Cargo.lock index 933343be..5b7a5337 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1205,6 +1205,7 @@ dependencies = [ "arboard", "clap", "difftastic", + "diffy-compare", "diffy-diff", "diffy-semantic", "diffy-syntax", @@ -1250,6 +1251,8 @@ dependencies = [ "diffy-diff", "diffy-semantic", "diffy-syntax", + "gix-imara-diff", + "tracing", ] [[package]] @@ -5739,6 +5742,7 @@ version = "0.3.7" dependencies = [ "clap", "diffy", + "diffy-compare", "diffy-diff", "diffy-semantic", "diffy-syntax", diff --git a/Cargo.toml b/Cargo.toml index a2450aca..425abbd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ parking_lot = "0.12" libc = "0.2" tempfile = "3.14" webbrowser = "1.0" +diffy-compare = { path = "crates/diffy-compare" } diffy-diff = { path = "crates/diffy-diff" } diffy-semantic = { path = "crates/diffy-semantic" } diffy-syntax = { path = "crates/diffy-syntax", features = ["download"] } diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml index 8c6f51fb..f5419e38 100644 --- a/crates/diffy-compare/Cargo.toml +++ b/crates/diffy-compare/Cargo.toml @@ -13,6 +13,11 @@ publish = false diffy-diff = { path = "../diffy-diff" } diffy-semantic = { path = "../diffy-semantic" } diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } +# The line diff gix runs (histogram plus Git's slider heuristics), the same +# crate and version gix already pulls into the app, so line diffs match Git +# and the app links one copy. +imara-diff = { package = "gix-imara-diff", version = "0.2.1", default-features = false } +tracing = "0.1.44" [features] default = [] diff --git a/crates/diffy-compare/src/document.rs b/crates/diffy-compare/src/document.rs new file mode 100644 index 00000000..d074a953 --- /dev/null +++ b/crates/diffy-compare/src/document.rs @@ -0,0 +1,369 @@ +//! Semantic rows to a [`DiffDocument`]: hunks, aligned rows and spans. + +use std::collections::HashSet; +use std::ops::Range; + +use diffy_diff::{ + AlignedRow, ChangeIntensity, DiffDocument, DocumentBuilder, FileBuilder, FileMeta, + LineGroupSpan, SemanticSpan, Side, SourceCoverage, TextStore, +}; +use diffy_semantic::{Move, OutlineItem, Rename, Row}; + +use crate::line::line_changes; +use crate::{Error, Result}; + +/// What a semantic diff knows beyond its rows. +#[derive(Default)] +pub struct Semantics { + pub outline: Vec, + pub renames: Vec, + pub moves: Vec, + pub groups: Vec, +} + +/// A file shown whole on one `side`: added or deleted. +pub(crate) fn whole_file( + meta: FileMeta, + old: TextStore, + new: TextStore, + side: Side, +) -> Result { + let lines = match side { + Side::Old => old.line_count(), + Side::New => new.line_count(), + }; + let (old, new) = match side { + Side::Old => (old, TextStore::default()), + Side::New => (TextStore::default(), new), + }; + build(meta, old, new, |file| { + if lines > 0 { + match side { + Side::Old => file.hunk(0, 0).change(lines, 0), + Side::New => file.hunk(0, 0).change(0, lines), + }; + } + }) +} + +pub(crate) fn build( + meta: FileMeta, + old: TextStore, + new: TextStore, + shape: impl FnOnce(&mut FileBuilder<'_>), +) -> Result { + let mut builder = DocumentBuilder::new(); + let mut file = builder.file(meta, SourceCoverage::Full, old, new); + shape(&mut file); + file.finish() + .map_err(|error| Error::Build(error.to_string()))?; + Ok(builder.build()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Context, + Change, +} + +#[derive(Clone, Copy)] +struct Segment { + kind: Kind, + old: u32, + new: u32, + shown: bool, +} + +struct HunkRange { + old: Range, + new: Range, + segments: Range, +} + +/// Drops rows' lines past the end of a side and spans that are empty, +/// out of bounds, split a character, or overlap an earlier span. +pub(crate) fn clean_rows(chunks: Vec>, old: &TextStore, new: &TextStore) -> Vec> { + let side = |store: &TextStore, line: Option, spans: Vec| { + let line = line.filter(|&l| l < store.line_count()); + let spans = line + .and_then(|l| store.display_line(l)) + .map(|text| clean_spans(spans, text)) + .unwrap_or_default(); + (line, spans) + }; + chunks + .into_iter() + .map(|chunk| { + chunk + .into_iter() + .map(|row| { + let [old_spans, new_spans] = row.spans; + let (old_line, old_spans) = side(old, row.old, old_spans); + let (new_line, new_spans) = side(new, row.new, new_spans); + Row { + old: old_line, + new: new_line, + spans: [old_spans, new_spans], + } + }) + .collect() + }) + .collect() +} + +/// `spans` clipped to `text`, sorted, without empty, overlapping, or +/// mid-character spans. +pub fn clean_spans(spans: Vec, text: &str) -> Vec { + let mut out: Vec = spans + .into_iter() + .filter_map(|span| { + let start = span.start as usize; + let end = (span.end as usize).min(text.len()); + (start < end && text.is_char_boundary(start) && text.is_char_boundary(end)).then_some( + SemanticSpan { + start: start as u32, + end: end as u32, + intensity: span.intensity, + }, + ) + }) + .collect(); + out.sort_by_key(|span| (span.start, span.end)); + let mut end = 0; + out.retain(|span| { + let keep = span.start >= end; + if keep { + end = span.end; + } + keep + }); + out +} + +/// Builds one file from semantic rows (cleaned, see [`clean_spans`]): +/// rows become hunks, and lines between rows that differ are line-diffed. +pub fn document_from_rows( + meta: FileMeta, + old: TextStore, + new: TextStore, + chunks: Vec>, + structural: bool, + semantics: Semantics, +) -> Result { + let segments = canonical_segments(&chunks, &old, &new); + let hunks = hunk_ranges(&segments); + let mut rows: Vec> = hunks.iter().map(|_| Vec::new()).collect(); + let mut spans: [Vec<(u32, Vec)>; 2] = [Vec::new(), Vec::new()]; + let mut seen = [HashSet::new(), HashSet::new()]; + for row in chunks.iter().flatten() { + let old_hunk = row.old.and_then(|line| find_hunk(&hunks, Side::Old, line)); + let new_hunk = row.new.and_then(|line| find_hunk(&hunks, Side::New, line)); + if let Some(h) = old_hunk.or(new_hunk) { + let range = &hunks[h]; + let aligned = AlignedRow { + old: row.old.filter(|line| range.old.contains(line)), + new: row.new.filter(|line| range.new.contains(line)), + }; + if aligned.old.is_some() || aligned.new.is_some() { + rows[h].push(aligned); + } + } + for (side, line, hunk) in [ + (Side::Old, row.old, old_hunk), + (Side::New, row.new, new_hunk), + ] { + let s = side as usize; + if let (Some(line), Some(_)) = (line, hunk) + && !row.spans[s].is_empty() + && seen[s].insert(line) + { + spans[s].push((line, row.spans[s].clone())); + } + } + } + build(meta, old, new, |file| { + file.structural(structural); + file.outline(semantics.outline); + file.renames(semantics.renames); + file.moves(semantics.moves); + file.groups(semantics.groups); + for (hunk, hunk_rows) in hunks.iter().zip(rows) { + file.hunk(hunk.old.start, hunk.new.start); + for segment in &segments[hunk.segments.clone()] { + match segment.kind { + Kind::Context => file.context(segment.old), + Kind::Change => file.change(segment.old, segment.new), + }; + } + for row in hunk_rows { + file.row(row); + } + } + for side in [Side::Old, Side::New] { + for (line, line_spans) in &spans[side as usize] { + file.spans(side, *line, line_spans); + } + } + }) +} + +fn is_context_row(row: &Row, old: &TextStore, new: &TextStore, lo: u32, ln: u32) -> bool { + old.line(lo) == new.line(ln) + && row + .spans + .iter() + .flatten() + .all(|span| span.intensity == ChangeIntensity::UnchangedContext) +} + +fn canonical_segments(chunks: &[Vec], old: &TextStore, new: &TextStore) -> Vec { + let mut segments = Vec::new(); + let (mut o, mut n) = (0u32, 0u32); + for chunk in chunks { + let mut first = true; + for row in chunk { + let lo = row.old.filter(|&line| line >= o); + let ln = row.new.filter(|&line| line >= n); + if lo.is_none() && ln.is_none() { + continue; + } + let (to, tn) = (lo.unwrap_or(o), ln.unwrap_or(n)); + push_unmentioned(&mut segments, old, new, o..to, n..tn, !first); + first = false; + let kind = match (lo, ln) { + (Some(a), Some(b)) if is_context_row(row, old, new, a, b) => Kind::Context, + _ => Kind::Change, + }; + segments.push(Segment { + kind, + old: u32::from(lo.is_some()), + new: u32::from(ln.is_some()), + shown: true, + }); + o = to + u32::from(lo.is_some()); + n = tn + u32::from(ln.is_some()); + } + } + push_unmentioned( + &mut segments, + old, + new, + o..old.line_count(), + n..new.line_count(), + false, + ); + segments +} + +fn push_unmentioned( + segments: &mut Vec, + old: &TextStore, + new: &TextStore, + old_lines: Range, + new_lines: Range, + inside: bool, +) { + let (old_len, new_len) = (old_lines.len() as u32, new_lines.len() as u32); + if old_len == 0 && new_len == 0 { + return; + } + let equal = old_len == new_len + && old_lines + .clone() + .zip(new_lines.clone()) + .all(|(a, b)| old.line(a) == new.line(b)); + if equal { + segments.push(Segment { + kind: Kind::Context, + old: old_len, + new: new_len, + shown: inside, + }); + return; + } + let changes = line_changes( + lines_bytes(old, old_lines.clone()), + lines_bytes(new, new_lines.clone()), + ); + let mut pieces = Vec::with_capacity(changes.len() * 2 + 1); + let (mut o, mut n) = (0u32, 0u32); + for change in &changes { + if change.old.start > o { + pieces.push((Kind::Context, change.old.start - o, change.new.start - n)); + } + pieces.push(( + Kind::Change, + change.old.len() as u32, + change.new.len() as u32, + )); + o = change.old.end; + n = change.new.end; + } + if o < old_len { + pieces.push((Kind::Context, old_len - o, new_len - n)); + } + let last = pieces.len().saturating_sub(1); + for (index, (kind, old, new)) in pieces.into_iter().enumerate() { + let edge = index == 0 || index == last; + segments.push(Segment { + kind, + old, + new, + shown: kind == Kind::Change || !edge || inside, + }); + } +} + +fn lines_bytes(store: &TextStore, lines: Range) -> &[u8] { + let text = store.as_str(); + let offset = |line: u32| { + store + .line_range(line) + .map_or(text.len(), |range| range.start) + }; + &text.as_bytes()[offset(lines.start)..offset(lines.end)] +} + +fn hunk_ranges(segments: &[Segment]) -> Vec { + let mut hunks = Vec::new(); + let (mut o, mut n) = (0u32, 0u32); + let mut open: Option<(u32, u32, usize, bool)> = None; + for (index, segment) in segments.iter().enumerate() { + let in_hunk = segment.shown || segment.kind == Kind::Change; + match (&mut open, in_hunk) { + (None, true) => open = Some((o, n, index, segment.kind == Kind::Change)), + (Some(state), true) => state.3 |= segment.kind == Kind::Change, + (Some(_), false) => close(&mut hunks, open.take(), o, n, index), + (None, false) => {} + } + o += segment.old; + n += segment.new; + } + close(&mut hunks, open, o, n, segments.len()); + hunks +} + +fn close( + hunks: &mut Vec, + open: Option<(u32, u32, usize, bool)>, + o: u32, + n: u32, + end: usize, +) { + if let Some((start_o, start_n, start, true)) = open { + hunks.push(HunkRange { + old: start_o..o, + new: start_n..n, + segments: start..end, + }); + } +} + +fn find_hunk(hunks: &[HunkRange], side: Side, line: u32) -> Option { + let range = |h: &HunkRange| match side { + Side::Old => h.old.clone(), + Side::New => h.new.clone(), + }; + let index = hunks.partition_point(|h| range(h).end <= line); + (index < hunks.len() && range(&hunks[index]).contains(&line)).then_some(index) +} diff --git a/crates/diffy-compare/src/file.rs b/crates/diffy-compare/src/file.rs new file mode 100644 index 00000000..96b679ca --- /dev/null +++ b/crates/diffy-compare/src/file.rs @@ -0,0 +1,156 @@ +//! One file's chain: trivial cases, then the semantic diff under its +//! budget, else a line diff. + +use std::time::{Duration, Instant}; + +use diffy_diff::{DiffDocument, FileMeta, FileStatus, LineGroupSpan, Side, TextStore}; +use diffy_semantic::Limits; +use diffy_syntax::LanguageId; + +use crate::document::{Semantics, build, clean_rows, document_from_rows, whole_file}; +use crate::grammar::SemanticGrammar; +use crate::line::{binary_document, line_diff}; +use crate::{Fallback, FileDiff, FileInput, Options, Result, is_binary, semantic_tag}; + +/// A file before any diff runs: either already done (binary, added, +/// deleted, unchanged) or two texts to diff. +pub enum Prepared { + Done(Box), + Text { + meta: FileMeta, + old: TextStore, + new: TextStore, + }, +} + +/// Settles the files no diff algorithm needs to look at. +pub fn prepare(input: &FileInput) -> Result { + let meta = input.meta(); + if is_binary(&input.old) || is_binary(&input.new) { + return Ok(Prepared::Done(Box::new(binary_document(meta)?))); + } + let old = TextStore::from_bytes(&input.old); + let new = TextStore::from_bytes(&input.new); + let done = if input.status == FileStatus::Added || input.old_path.is_none() { + whole_file(meta, old, new, Side::New)? + } else if input.status == FileStatus::Deleted || input.new_path.is_none() { + whole_file(meta, old, new, Side::Old)? + } else if input.old == input.new { + build(meta, old, new, |_| {})? + } else { + return Ok(Prepared::Text { meta, old, new }); + }; + Ok(Prepared::Done(Box::new(done))) +} + +pub(crate) fn diff_one( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + let (meta, old, new) = match prepare(input)? { + Prepared::Done(document) => { + return Ok(FileDiff { + document: *document, + fallback: None, + }); + } + Prepared::Text { meta, old, new } => (meta, old, new), + }; + let line = |meta: FileMeta, fallback: Fallback| { + Ok(FileDiff { + document: line_diff(meta, &input.old, &input.new)?, + fallback: Some(fallback), + }) + }; + let Some(tag) = semantic_tag(input.display_path()) else { + return line(meta, Fallback::NoGrammar); + }; + if budget(options).is_none() { + return line(meta, Fallback::RequestBudget); + } + let Some(grammar) = grammars(&tag) else { + return line(meta, Fallback::GrammarUnavailable(tag.to_string())); + }; + // Loading the grammar can take a while the first time, so the budget + // is measured again after it. + let Some(max_time) = budget(options) else { + return line(meta, Fallback::RequestBudget); + }; + let limits = Limits { + max_time, + ..Limits::default() + }; + let embed = |tree: &diffy_semantic::Tree, source: &str| { + let Some(injector) = &grammar.injector else { + return Vec::new(); + }; + // Regions repeat a few languages many times (every `{expression}` + // of a Svelte template), so each is looked up once. + let mut seen: Vec<(LanguageId, Option)> = Vec::new(); + let mut out = Vec::new(); + for (language, range) in injector.find(tree, source) { + let at = match seen.iter().position(|(l, _)| *l == language) { + Some(at) => at, + None => { + let found = grammars(&language); + seen.push((language, found)); + seen.len() - 1 + } + }; + if let Some(embedded) = &seen[at].1 { + out.push(diffy_semantic::Embedded { + range, + language: embedded.language.clone(), + rules: embedded.rules.clone(), + name: embedded.name.clone(), + }); + } + } + out + }; + let diff = diffy_semantic::diff_with( + &grammar.language, + &grammar.rules, + old.as_str(), + new.as_str(), + &limits, + options.vet, + Some(&embed), + ); + match diff { + Ok(diff) => { + let chunks = clean_rows(diff.chunks, &old, &new); + let semantics = Semantics { + outline: diff.outline, + renames: diff.renames, + moves: diff.moves, + groups: diff + .groups + .into_iter() + .map(|g| LineGroupSpan { + old: g.old, + new: g.new, + }) + .collect(), + }; + Ok(FileDiff { + document: document_from_rows(meta, old, new, chunks, true, semantics)?, + fallback: None, + }) + } + Err(fallback) => line(meta, Fallback::Semantic(fallback)), + } +} + +/// The semantic time budget for a file starting now, or `None` once the +/// request's deadline has passed. +fn budget(options: &Options<'_>) -> Option { + let Some(deadline) = options.deadline else { + return Some(options.file_budget); + }; + deadline + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + .map(|left| left.min(options.file_budget)) +} diff --git a/crates/diffy-compare/src/grammar.rs b/crates/diffy-compare/src/grammar.rs new file mode 100644 index 00000000..be0212bf --- /dev/null +++ b/crates/diffy-compare/src/grammar.rs @@ -0,0 +1,71 @@ +//! Grammars for the semantic diff, from the caller's store. + +use std::sync::{Arc, Mutex, PoisonError}; + +use diffy_semantic::{Queries, Rules}; +use diffy_syntax::{GrammarQueries, GrammarStore, Injector, LanguageId}; + +/// What the semantic diff needs from one language's grammar. +pub struct SemanticGrammar { + pub name: Arc, + pub language: diffy_semantic::Language, + pub rules: Arc, + pub injector: Option, +} + +/// The grammar `store` has for `tag`, with its semantic rules compiled +/// once per loaded grammar. +pub fn semantic_grammar(store: &GrammarStore, tag: &LanguageId) -> Option { + static RULES: Mutex, Arc)>> = Mutex::new(Vec::new()); + let (language, queries) = store.parser(tag.as_str())?; + let cached = |rules: &[(Arc, Arc)]| { + rules + .iter() + .find(|(highlights, _)| Arc::ptr_eq(highlights, &queries.highlights)) + .map(|(_, rules)| rules.clone()) + }; + let found = cached(&RULES.lock().unwrap_or_else(PoisonError::into_inner)); + let rules = match found { + Some(rules) => rules, + None => { + let rules = Arc::new(compile_rules(&language, &queries)); + let mut all = RULES.lock().unwrap_or_else(PoisonError::into_inner); + match cached(&all) { + Some(rules) => rules, + None => { + all.push((queries.highlights.clone(), rules.clone())); + rules + } + } + } + }; + Some(SemanticGrammar { + injector: store.injector(tag.as_str()), + name: queries.language, + language, + rules, + }) +} + +fn compile_rules(language: &diffy_semantic::Language, grammar: &GrammarQueries) -> Rules { + let queries = Queries { + highlights: &grammar.highlights, + locals: grammar.locals.as_deref(), + }; + let own = [ + diffy_semantic::semantic_query(&grammar.language), + grammar.semantic.as_deref(), + ]; + for semantic in own.into_iter().flatten() { + match Rules::new(language, semantic, &queries) { + Ok(rules) => return rules, + Err(error) => tracing::warn!( + target: "diffy::semantic", + language = %grammar.language, + %error, + "semantic query does not compile" + ), + } + } + Rules::derived(language, &queries) +} diff --git a/crates/diffy-compare/src/lib.rs b/crates/diffy-compare/src/lib.rs index db14cdb4..8b2252ac 100644 --- a/crates/diffy-compare/src/lib.rs +++ b/crates/diffy-compare/src/lib.rs @@ -16,18 +16,30 @@ //! With the `serde` feature, [`wire`] projects results into a versioned //! display format that non-Rust clients render from. -// CONTRACT SKELETON: the types and signatures below are the agreed API. -// Bodies marked `todo!` are filled in by the compare-core change. - +use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::time::{Duration, Instant}; -pub use diffy_diff::{DiffDocument, FileStatus}; +pub use diffy_diff::{DiffDocument, FileMeta, FileStatus}; pub use diffy_syntax::{GrammarStore, LanguageId}; +mod document; +mod file; +mod grammar; +mod line; +mod run; +#[cfg(test)] +mod tests; #[cfg(feature = "serde")] pub mod wire; +#[doc(hidden)] +pub use document::{Semantics, clean_spans, document_from_rows}; +#[doc(hidden)] +pub use file::{Prepared, prepare}; +pub use grammar::{SemanticGrammar, semantic_grammar}; +pub use line::{CONTEXT_LINES, binary_document, line_diff}; + /// The semantic diff's default time budget per file. pub const FILE_BUDGET: Duration = Duration::from_millis(400); @@ -51,6 +63,16 @@ impl FileInput { .or(self.old_path.as_deref()) .unwrap_or_default() } + + /// The document metadata for this file: paths and status. + pub fn meta(&self) -> FileMeta { + FileMeta { + old_path: self.old_path.as_deref().map(Arc::from), + new_path: self.new_path.as_deref().map(Arc::from), + status: self.status, + ..FileMeta::default() + } + } } /// Why a file got a line diff instead of a semantic one. @@ -163,15 +185,35 @@ pub type Result = std::result::Result; /// Diffs one file on the calling thread. pub fn diff_file(input: &FileInput, options: &Options<'_>) -> Result { - let _ = (input, options); - todo!("compare core") + diff_file_with(input, options, &|tag| { + semantic_grammar(options.grammars, tag) + }) +} + +/// [`diff_file`] with grammars from `grammars` instead of +/// [`Options::grammars`], for tools that wait on or swap grammars. +#[doc(hidden)] +pub fn diff_file_with( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + if run::cancelled(options) { + return Err(Error::Cancelled); + } + let diff = run::observed(input, options, grammars)?; + if let Some(observer) = options.observer { + observer.progress(1, 1); + } + Ok(diff) } /// Diffs every file, in parallel when there are several, and returns the /// results in input order. pub fn diff_files(inputs: &[FileInput], options: &Options<'_>) -> Result> { - let _ = (inputs, options); - todo!("compare core") + run::diff_all(inputs, options, &|tag| { + semantic_grammar(options.grammars, tag) + }) } /// The language tag the semantic diff uses for `path`: JavaScript is diff --git a/crates/diffy-compare/src/line.rs b/crates/diffy-compare/src/line.rs new file mode 100644 index 00000000..2fd3e606 --- /dev/null +++ b/crates/diffy-compare/src/line.rs @@ -0,0 +1,55 @@ +//! Line diffs: what a file gets when the semantic diff does not run. + +use diffy_diff::{ + DiffDocument, DocumentBuilder, FileMeta, SourceCoverage, TextStore, diff_from_changes, + myers::Change, +}; +use imara_diff::{Algorithm, Diff, InternedInput}; + +use crate::{Error, Result, is_binary}; + +/// Unchanged lines shown around each change in a line diff. +pub const CONTEXT_LINES: u32 = 3; + +/// A line diff of `old` and `new` as one file described by `meta`, or a +/// binary placeholder when either side looks binary. +pub fn line_diff(mut meta: FileMeta, old: &[u8], new: &[u8]) -> Result { + if is_binary(old) || is_binary(new) { + return binary_document(meta); + } + let changes = line_changes(old, new); + let (old, new) = (TextStore::from_bytes(old), TextStore::from_bytes(new)); + meta.binary = false; + diff_from_changes(meta, old, new, &changes, CONTEXT_LINES) + .map_err(|error| Error::Build(error.to_string())) +} + +/// A document for a binary file: its metadata and no text. +pub fn binary_document(mut meta: FileMeta) -> Result { + meta.binary = true; + let mut builder = DocumentBuilder::new(); + builder + .file( + meta, + SourceCoverage::PatchOnly, + TextStore::default(), + TextStore::default(), + ) + .finish() + .map_err(|error| Error::Build(error.to_string()))?; + Ok(builder.build()) +} + +/// Histogram line diff with Git's slider heuristics, as `git diff` and +/// gix compute it. +pub(crate) fn line_changes(old: &[u8], new: &[u8]) -> Vec { + let input = InternedInput::new(old, new); + let mut diff = Diff::compute(Algorithm::Histogram, &input); + diff.postprocess_lines(&input); + diff.hunks() + .map(|hunk| Change { + old: hunk.before, + new: hunk.after, + }) + .collect() +} diff --git a/crates/diffy-compare/src/run.rs b/crates/diffy-compare/src/run.rs new file mode 100644 index 00000000..3c8a917f --- /dev/null +++ b/crates/diffy-compare/src/run.rs @@ -0,0 +1,112 @@ +//! Running files on worker threads. + +use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use diffy_syntax::LanguageId; + +use crate::file::diff_one; +use crate::grammar::SemanticGrammar; +use crate::{Error, FileDiff, FileInput, Options, Result}; + +const MAX_WORKERS: usize = 4; +const MIN_FILES_FOR_PARALLEL: usize = 4; +/// Tree-sitter parsing and the semantic matcher recurse deeply on nested +/// sources, more than a default thread stack allows. +const WORKER_STACK_BYTES: usize = 16 << 20; + +type Grammars<'g> = dyn Fn(&LanguageId) -> Option + Sync + 'g; + +pub(crate) fn cancelled(options: &Options<'_>) -> bool { + options + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) +} + +/// Diffs `input` under the observer's [`Observer::file`] hook. +/// +/// [`Observer::file`]: crate::Observer::file +pub(crate) fn observed( + input: &FileInput, + options: &Options<'_>, + grammars: &dyn Fn(&LanguageId) -> Option, +) -> Result { + let Some(observer) = options.observer else { + return diff_one(input, options, grammars); + }; + let mut out = None; + observer.file(input.display_path(), &mut || { + out = Some(diff_one(input, options, grammars)); + }); + match out { + Some(out) => out, + // The observer did not run the work; it still has to happen. + None => diff_one(input, options, grammars), + } +} + +/// Diffs `inputs` on worker threads, which take the next file in order +/// until none remain or the request is cancelled. +pub(crate) fn diff_all( + inputs: &[FileInput], + options: &Options<'_>, + grammars: &Grammars<'_>, +) -> Result> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + let workers = match options.workers { + 0 => worker_count(inputs.len()), + workers => workers.clamp(1, inputs.len()), + }; + let total = inputs.len() as u32; + let next = AtomicUsize::new(0); + let seen = AtomicU32::new(0); + let mut done = std::thread::scope(|scope| { + let handles = (0..workers) + .map(|index| { + std::thread::Builder::new() + .name(format!("diffy-semantic-{index}")) + .stack_size(WORKER_STACK_BYTES) + .spawn_scoped(scope, || { + let mut done = Vec::new(); + while !cancelled(options) { + let at = next.fetch_add(1, Ordering::Relaxed); + let Some(input) = inputs.get(at) else { + break; + }; + done.push((at, observed(input, options, grammars))); + let seen = seen.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if let Some(observer) = options.observer { + observer.progress(seen, total); + } + } + done + }) + .map_err(|error| Error::Worker(format!("setup failed: {error}"))) + }) + .collect::>>()?; + let mut done = Vec::with_capacity(inputs.len()); + for handle in handles { + let part = handle + .join() + .map_err(|_| Error::Worker("a worker panicked".to_owned()))?; + done.extend(part); + } + Ok::<_, Error>(done) + })?; + if done.len() < inputs.len() { + return Err(Error::Cancelled); + } + done.sort_unstable_by_key(|(at, _)| *at); + done.into_iter().map(|(_, diff)| diff).collect() +} + +fn worker_count(files: usize) -> usize { + if files < MIN_FILES_FOR_PARALLEL { + return 1; + } + let available = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + files.min(available).clamp(1, MAX_WORKERS) +} diff --git a/crates/diffy-compare/src/tests.rs b/crates/diffy-compare/src/tests.rs new file mode 100644 index 00000000..4a42b467 --- /dev/null +++ b/crates/diffy-compare/src/tests.rs @@ -0,0 +1,303 @@ +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::{Duration, Instant}; + +use diffy_diff::{ + BlockKind, ChangeIntensity as Intensity, DiffDocument, FileMeta, FileStatus, SemanticSpan, + Side, TextStore, +}; +use diffy_semantic::Row; + +use crate::document::{Semantics, clean_rows, document_from_rows}; +use crate::{ + Error, Fallback, FileInput, GrammarStore, Observer, Options, diff_file, diff_files, line_diff, +}; + +fn span(start: u32, end: u32, intensity: Intensity) -> SemanticSpan { + SemanticSpan { + start, + end, + intensity, + } +} + +fn line( + old: Option, + new: Option, + old_spans: Vec, + new_spans: Vec, +) -> Row { + Row { + old, + new, + spans: [old_spans, new_spans], + } +} + +fn rows_document(old: &str, new: &str, chunks: Vec>) -> DiffDocument { + let (old, new) = (TextStore::new(old), TextStore::new(new)); + let chunks = clean_rows(chunks, &old, &new); + document_from_rows(meta(), old, new, chunks, true, Semantics::default()).unwrap() +} + +fn meta() -> FileMeta { + FileMeta { + old_path: Some("src/lib.rs".into()), + new_path: Some("src/lib.rs".into()), + status: FileStatus::Modified, + ..FileMeta::default() + } +} + +fn hunks(doc: &DiffDocument) -> Vec<(u32, u32, u32, u32)> { + let h = doc.hunks(); + (0..doc.hunk_count() as usize) + .map(|i| (h.old_start[i], h.old_len[i], h.new_start[i], h.new_len[i])) + .collect() +} + +fn modified(path: &str, old: &str, new: &str) -> FileInput { + FileInput { + status: FileStatus::Modified, + old_path: Some(path.to_owned()), + new_path: Some(path.to_owned()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + } +} + +const OLD_RS: &str = "fn answer() {\n old();\n}\n"; +const NEW_RS: &str = "fn answer() {\n new();\n}\n"; + +#[test] +fn semantic_rows_keep_intensities_pairing_and_full_sources() { + let old = "fn a() {}\nlet x = \"foo\";\nkeep();\n"; + let new = "fn a() {}\nlet x = \"bar\";\nextra();\nkeep();\n"; + let doc = rows_document( + old, + new, + vec![vec![ + line(Some(0), Some(0), vec![], vec![]), + line( + Some(1), + Some(1), + vec![ + span(8, 9, Intensity::UnchangedContext), + span(9, 12, Intensity::NovelWord), + span(12, 13, Intensity::UnchangedContext), + ], + vec![span(9, 12, Intensity::NovelWord)], + ), + line(None, Some(2), vec![], vec![span(0, 7, Intensity::Novel)]), + line(Some(2), Some(3), vec![], vec![]), + ]], + ); + doc.verify_integrity().unwrap(); + assert_eq!(hunks(&doc), [(1, 3, 1, 4)]); + assert_eq!( + doc.blocks().kind, + [BlockKind::Context, BlockKind::Change, BlockKind::Context] + ); + assert_eq!(doc.stats(0).known(), Some((2, 1))); + assert!(doc.prefers_structural(0)); + let intensities: Vec<_> = doc + .semantic_spans(0, Side::Old, 1) + .iter() + .map(|s| s.intensity) + .collect(); + assert_eq!( + intensities, + [ + Intensity::UnchangedContext, + Intensity::NovelWord, + Intensity::UnchangedContext + ] + ); + assert_eq!( + doc.semantic_spans(0, Side::New, 2)[0].intensity, + Intensity::Novel + ); + let rows: Vec<_> = doc.aligned_rows(0).iter().map(|r| (r.old, r.new)).collect(); + assert_eq!( + rows, + [ + (Some(0), Some(0)), + (Some(1), Some(1)), + (None, Some(2)), + (Some(2), Some(3)) + ] + ); + assert_eq!(doc.text(0, Side::New).line_count(), 4); +} + +#[test] +fn lines_that_differ_between_chunks_become_changes() { + let old = "a\nb\nsame\nc\nd\n"; + let new = "A\nb\nsame2\nc\nD\n"; + let doc = rows_document( + old, + new, + vec![ + vec![line( + Some(0), + Some(0), + vec![span(0, 1, Intensity::Novel)], + vec![span(0, 1, Intensity::Novel)], + )], + vec![line( + Some(4), + Some(4), + vec![span(0, 1, Intensity::Novel)], + vec![span(0, 1, Intensity::Novel)], + )], + ], + ); + doc.verify_integrity().unwrap(); + assert_eq!(hunks(&doc), [(1, 1, 1, 1), (3, 1, 3, 1), (5, 1, 5, 1)]); + assert_eq!(doc.stats(0).known(), Some((3, 3))); +} + +#[test] +fn separate_chunks_stay_separate_hunks_and_bad_spans_are_dropped() { + let old = "a\n1\n2\n3\n4\nb\n"; + let new = "x\n1\n2\n3\n4\ny\n"; + let doc = rows_document( + old, + new, + vec![ + vec![line( + Some(0), + Some(0), + vec![span(0, 9, Intensity::Novel)], + vec![], + )], + vec![ + line(Some(5), Some(5), vec![], vec![span(3, 2, Intensity::Novel)]), + line(Some(6), Some(6), vec![], vec![]), + ], + ], + ); + assert_eq!(hunks(&doc), [(1, 1, 1, 1), (6, 1, 6, 1)]); + assert_eq!(doc.semantic_spans(0, Side::Old, 0)[0].end, 1); + assert!(doc.semantic_spans(0, Side::New, 5).is_empty()); +} + +#[test] +fn added_files_skip_the_semantic_diff() { + let store = GrammarStore::none(); + let diff = diff_file( + &FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some("src/new.rs".to_owned()), + old: Vec::new(), + new: b"fn new() {}\nlet x = 1;\n".to_vec(), + }, + &Options::new(&store), + ) + .unwrap(); + let doc = diff.document; + assert_eq!(diff.fallback, None); + assert_eq!(hunks(&doc), [(0, 0, 1, 2)]); + assert_eq!(doc.text(0, Side::New).line(1), Some("let x = 1;")); + assert_eq!(doc.summary(0).status, FileStatus::Added); +} + +#[test] +fn files_without_a_grammar_get_line_diffs() { + let store = GrammarStore::none(); + let input = modified("src/lib.rs", "a\nb\n", "a\nc\n"); + let diff = diff_file(&input, &Options::new(&store)).unwrap(); + let fallback = diff.fallback.unwrap(); + assert!(matches!(fallback, Fallback::GrammarUnavailable(_))); + assert!(fallback.to_string().ends_with(" grammar unavailable")); + assert_eq!(hunks(&diff.document), [(1, 2, 1, 2)]); +} + +#[test] +fn rust_files_get_a_semantic_diff_within_the_budget() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let mut options = Options::new(&store); + options.deadline = Some(Instant::now() + Duration::from_secs(60)); + let diff = diff_file(&modified("src/lib.rs", OLD_RS, NEW_RS), &options).unwrap(); + assert_eq!(diff.fallback, None); + assert!(diff.document.prefers_structural(0)); + assert!(!diff.document.semantic_spans(0, Side::New, 1).is_empty()); +} + +#[test] +fn files_after_the_deadline_get_line_diffs() { + // No grammar is needed: the deadline is checked before one loads. + let store = GrammarStore::none(); + let mut options = Options::new(&store); + options.deadline = Some(Instant::now()); + let inputs = [ + modified("src/lib.rs", OLD_RS, NEW_RS), + FileInput { + status: FileStatus::Added, + old_path: None, + new_path: Some("src/new.rs".to_owned()), + new: b"fn new() {}\n".to_vec(), + ..FileInput::default() + }, + modified("src/main.rs", OLD_RS, NEW_RS), + ]; + let diffs = diff_files(&inputs, &options).unwrap(); + let fallbacks: Vec<_> = diffs.iter().map(|diff| diff.fallback.clone()).collect(); + assert_eq!( + fallbacks, + [ + Some(Fallback::RequestBudget), + None, + Some(Fallback::RequestBudget) + ] + ); + let line = line_diff(inputs[0].meta(), OLD_RS.as_bytes(), NEW_RS.as_bytes()).unwrap(); + assert_eq!(hunks(&diffs[0].document), hunks(&line)); + assert!(!diffs[0].document.prefers_structural(0)); +} + +#[test] +fn cancelled_requests_stop() { + let store = GrammarStore::none(); + let cancel = AtomicBool::new(true); + let mut options = Options::new(&store); + options.cancel = Some(&cancel); + let input = modified("src/lib.rs", OLD_RS, NEW_RS); + assert_eq!(diff_file(&input, &options).unwrap_err(), Error::Cancelled); + assert_eq!( + diff_files(&[input.clone(), input], &options).unwrap_err(), + Error::Cancelled + ); +} + +struct CancelAfterFirst<'a> { + cancel: &'a AtomicBool, + files: AtomicU32, +} + +impl Observer for CancelAfterFirst<'_> { + fn file(&self, _path: &str, work: &mut dyn FnMut()) { + self.files.fetch_add(1, Ordering::Relaxed); + work(); + self.cancel.store(true, Ordering::Relaxed); + } +} + +#[test] +fn cancelling_mid_request_stops_picking_up_files() { + let store = GrammarStore::none(); + let cancel = AtomicBool::new(false); + let observer = CancelAfterFirst { + cancel: &cancel, + files: AtomicU32::new(0), + }; + let mut options = Options::new(&store); + options.cancel = Some(&cancel); + options.workers = 1; + options.observer = Some(&observer); + let inputs = vec![modified("a.txt", "a\n", "b\n"); 8]; + assert_eq!(diff_files(&inputs, &options).unwrap_err(), Error::Cancelled); + assert_eq!(observer.files.load(Ordering::Relaxed), 1); +} diff --git a/src/app/viewport/pipeline/state.rs b/src/app/viewport/pipeline/state.rs index c6d57110..c1d7b1fa 100644 --- a/src/app/viewport/pipeline/state.rs +++ b/src/app/viewport/pipeline/state.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::rc::Rc; use std::sync::Arc; +use diffy_compare::semantic_tag; use diffy_diff::{OutlineItem, Placeholder, SlotState, SourceCoverage, SourcePurpose}; use crate::contracts::commands::{SettingsSnapshot, ToastKind, ToastUpdate}; @@ -20,7 +21,6 @@ pub use crate::contracts::viewport::pipeline::{ WORKING_SET_LINES, }; use crate::core::compare::git::bucket_refs; -use crate::core::compare::semantic::semantic_tag; use crate::core::compare::{CompareEntry, ComparePhase, RendererKind}; use crate::core::vcs::model::{ChangeBucket, VcsCompareSpec}; use crate::platform::persistence::{PersistedCompare, Settings}; diff --git a/src/core/compare/git.rs b/src/core/compare/git.rs index 14754471..4cef2dab 100644 --- a/src/core/compare/git.rs +++ b/src/core/compare/git.rs @@ -1,9 +1,7 @@ use std::sync::Arc; -use diffy_diff::{ - DiffDocument, DiffStats, DocumentBuilder, FileMeta, FileStatus, ObjectId, SourceCoverage, - TextStore, diff_from_changes, myers::Change, -}; +use diffy_compare::{FileInput, binary_document, is_binary, line_diff}; +use diffy_diff::{DiffDocument, DiffStats, FileMeta, FileStatus, ObjectId}; use gix::bstr::{BStr, ByteSlice}; use gix::diff::blob::{Algorithm, InternedInput, diff_with_slider_heuristics}; @@ -21,7 +19,6 @@ use crate::core::vcs::model::ChangeBucket; const LOADING_FILE_EMIT_STRIDE: usize = 16; const LINE_STATS_MAX_WORKERS: usize = 8; const DEFERRED_STATS_MIN_FILES_PER_WORKER: usize = 16; -pub const CONTEXT_LINES: u32 = 3; pub fn compare( spec: &CompareSpec, @@ -256,12 +253,12 @@ fn worktree_document( }); let (old, new) = (old.unwrap_or_default(), new.unwrap_or_default()); if renderer == RendererKind::Semantic { - let changed = crate::core::compare::semantic::ChangedPath { + let changed = FileInput { status: meta.status, old_path: meta.old_path.as_deref().map(str::to_owned), new_path: meta.new_path.as_deref().map(str::to_owned), - old_content: old, - new_content: new, + old, + new, }; let mut doc = crate::core::compare::semantic::document_from_changed(&changed)?; doc.set_identity( @@ -272,7 +269,7 @@ fn worktree_document( return Ok(Some(doc)); } let _ = renderer; - document_from_bytes(meta, &old, &new).map(Some) + Ok(Some(line_diff(meta, &old, &new)?)) } pub fn compare_path( @@ -302,7 +299,7 @@ fn push_untracked(output: &mut CompareOutput, git: &GitService, only: Option<&st status: FileStatus::Added, ..FileMeta::default() }; - output.push_document(Arc::new(document_from_bytes(meta, &[], &new)?)); + output.push_document(Arc::new(line_diff(meta, &[], &new)?)); } Ok(()) } @@ -310,7 +307,7 @@ fn push_untracked(output: &mut CompareOutput, git: &GitService, only: Option<&st pub fn compare_entry(entry: &CompareEntry, git: &GitService) -> Result> { let meta = entry.file_meta(); if entry.is_binary() { - return binary_document(meta).map(Some); + return Ok(Some(binary_document(meta)?)); } let loadable = match entry.status() { FileStatus::Added => entry.new_id.is_some(), @@ -323,7 +320,7 @@ pub fn compare_entry(entry: &CompareEntry, git: &GitService) -> Result Result { @@ -344,7 +341,7 @@ pub fn compare_text(left: &str, right: &str, display_path: &str) -> Result Result { - if is_binary_bytes(old) || is_binary_bytes(new) { - return binary_document(meta); - } - let changes = line_changes(old, new); - let (old, new) = (TextStore::from_bytes(old), TextStore::from_bytes(new)); - meta.binary = false; - diff_from_changes(meta, old, new, &changes, CONTEXT_LINES) - .map_err(|error| DiffyError::Parse(error.to_string())) -} - -pub(crate) fn binary_document(mut meta: FileMeta) -> Result { - meta.binary = true; - let mut builder = DocumentBuilder::new(); - builder - .file( - meta, - SourceCoverage::PatchOnly, - TextStore::default(), - TextStore::default(), - ) - .finish() - .map_err(|error| DiffyError::Parse(error.to_string()))?; - Ok(builder.build()) -} - -pub(crate) fn line_changes(old: &[u8], new: &[u8]) -> Vec { - let input = InternedInput::new(old, new); - diff_with_slider_heuristics(Algorithm::Histogram, &input) - .hunks() - .map(|hunk| Change { - old: hunk.before, - new: hunk.after, - }) - .collect() -} - pub(crate) fn line_stats(old: &[u8], new: &[u8]) -> (u32, u32) { let input = InternedInput::new(old, new); let diff = diff_with_slider_heuristics(Algorithm::Histogram, &input); @@ -469,7 +425,7 @@ fn output_from_gix_changes( continue; }; let meta = entry_from_gix_change(change).file_meta(); - let doc = document_from_bytes(meta, old.as_bytes(), new.as_bytes())?; + let doc = line_diff(meta, old.as_bytes(), new.as_bytes())?; output.push_document(Arc::new(doc)); } Ok(output) @@ -612,7 +568,7 @@ fn entry_stats_with_trees( let (Some(old), Some(new)) = (old, new) else { return Ok(None); }; - if is_binary_bytes(old.as_bytes()) || is_binary_bytes(new.as_bytes()) { + if is_binary(old.as_bytes()) || is_binary(new.as_bytes()) { return Ok(Some(known(0, 0))); } let (additions, deletions) = line_stats(old.as_bytes(), new.as_bytes()); @@ -840,10 +796,6 @@ pub(crate) fn load_gix_blob_id( )) } -pub(crate) fn is_binary_bytes(bytes: &[u8]) -> bool { - bytes.iter().take(8192).any(|byte| *byte == 0) -} - pub(crate) fn deferred_stats_worker_count(file_count: usize) -> usize { let available = std::thread::available_parallelism() .map(usize::from) diff --git a/src/core/compare/semantic.rs b/src/core/compare/semantic.rs index 5a80931e..37e4e22f 100644 --- a/src/core/compare/semantic.rs +++ b/src/core/compare/semantic.rs @@ -1,64 +1,31 @@ use std::collections::HashMap; -use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use std::time::Duration; -use diffy_diff::{ - AlignedRow, ChangeIntensity, DiffDocument, DocumentBuilder, FileMeta, FileStatus, SemanticSpan, - Side, SourceCoverage, TextStore, -}; -use diffy_semantic::{Limits, Move, OutlineItem, Queries, Rename, Row, Rules}; -use diffy_syntax::{GrammarQueries, GrammarStore, Injector, LanguageId}; +use diffy_compare::{FileDiff, FileInput, Observer, Options}; +#[cfg(feature = "difftastic")] +use diffy_diff::{ChangeIntensity, FileMeta, SemanticSpan, TextStore}; +use diffy_diff::{DiffDocument, FileStatus}; +#[cfg(feature = "difftastic")] +use diffy_semantic::Row; #[cfg(feature = "difftastic")] use vendored_difftastic::{ ChangeIntensity as DftIntensity, ChangeSpan, DiffRequest, DiffStatus, SemanticDiffResult, }; -use crate::core::compare::git::{ - binary_document, document_from_bytes, is_binary_bytes, line_changes, resolve_compare_refs, -}; +use crate::core::compare::git::resolve_compare_refs; use crate::core::compare::output::{CompareEntry, CompareOutput}; use crate::core::compare::progress::{ComparePhase, ProgressSink}; use crate::core::compare::spec::CompareSpec; use crate::core::compare::stats::COMPARE_SUMMARY_FILE_LIMIT; -use crate::core::error::{DiffyError, Result}; +#[cfg(feature = "difftastic")] +use crate::core::error::DiffyError; +use crate::core::error::Result; use crate::core::syntax::grammar::Grammars; use crate::core::vcs::git::{ GitService, INDEX_REF, StatusItem, StatusScope, WORKDIR_REF, mentions, }; const LOADING_FILE_EMIT_STRIDE: u32 = 16; -const MAX_WORKERS: usize = 4; -const MIN_FILES_FOR_PARALLEL: usize = 4; -const WORKER_STACK_BYTES: usize = 16 << 20; -const FILE_TIME_BUDGET: Duration = Duration::from_millis(400); - -#[derive(Debug, Clone)] -pub struct ChangedPath { - pub status: FileStatus, - pub old_path: Option, - pub new_path: Option, - pub old_content: Vec, - pub new_content: Vec, -} - -impl ChangedPath { - fn display_path(&self) -> &str { - self.new_path - .as_deref() - .or(self.old_path.as_deref()) - .unwrap_or_default() - } - - fn meta(&self) -> FileMeta { - FileMeta { - old_path: self.old_path.as_deref().map(Arc::from), - new_path: self.new_path.as_deref().map(Arc::from), - status: self.status, - ..FileMeta::default() - } - } -} pub fn status_from_label(label: &str) -> FileStatus { match label.as_bytes().first() { @@ -125,12 +92,12 @@ pub fn compare_status_item(item: &StatusItem, git: &GitService) -> Result FileStatus::Deleted, _ => status_from_label(&item.status), }; - let changed = ChangedPath { + let changed = FileInput { status, old_path: old_content.as_ref().map(|_| old_path.to_owned()), new_path: new_content.as_ref().map(|_| item.path.clone()), - old_content: old_content.unwrap_or_default(), - new_content: new_content.unwrap_or_default(), + old: old_content.unwrap_or_default(), + new: new_content.unwrap_or_default(), }; let vet = rename_vet(git, old_ref.unwrap_or(new_ref), new_ref); compare_changed_paths(vec![changed], None, Some(&vet)) @@ -147,12 +114,12 @@ pub fn compare_text(left: &str, right: &str, display_path: &str) -> Result( } } +/// Attributes each file's diff to the energy monitor and reports +/// `LoadingFiles` progress every few files. +struct LoadingFiles<'a> { + reporter: Option<&'a dyn ProgressSink>, +} + +impl Observer for LoadingFiles<'_> { + fn file(&self, path: &str, work: &mut dyn FnMut()) { + let _work = crate::platform::energy::work(crate::platform::energy::Kind::Semantic) + .detail(path.to_owned()); + work(); + } + + fn progress(&self, seen: u32, total: u32) { + if let Some(r) = self.reporter + && (seen.is_multiple_of(LOADING_FILE_EMIT_STRIDE) || seen == total) + { + r.phase(ComparePhase::LoadingFiles { + files_seen: seen, + files_total: total, + }); + } + } +} + pub fn compare_changed_paths( - changed: Vec, + changed: Vec, reporter: Option<&dyn ProgressSink>, vet: Option<&diffy_semantic::Vet<'_>>, ) -> Result { - let files_total = changed.len() as u32; if let Some(r) = reporter { r.phase(ComparePhase::LoadingFiles { files_seen: 0, - files_total, + files_total: changed.len() as u32, }); } - let progress = AtomicU32::new(0); - let load = |changed: &ChangedPath| { - let doc = document_from_changed_vetted(changed, vet)?; - let seen = progress.fetch_add(1, Ordering::Relaxed).saturating_add(1); - if let Some(r) = reporter - && (seen.is_multiple_of(LOADING_FILE_EMIT_STRIDE) || seen == files_total) - { - r.phase(ComparePhase::LoadingFiles { - files_seen: seen, - files_total, - }); - } - Ok(Arc::new(doc)) - }; - let docs = load_parallel(&changed, worker_count(changed.len()), &load)?; + let store = Grammars::shared().store(); + let observer = LoadingFiles { reporter }; + let mut options = Options::new(&store); + options.vet = vet; + options.observer = Some(&observer); + let diffs = diffy_compare::diff_files(&changed, &options)?; let mut output = CompareOutput::default(); - for doc in docs { - output.push_document(doc); + for (input, diff) in changed.iter().zip(diffs) { + log_fallback(input, &diff); + output.push_document(Arc::new(diff.document)); } Ok(output) } -fn load_parallel( - changed: &[ChangedPath], - workers: usize, - load: &(dyn Fn(&ChangedPath) -> Result> + Sync), -) -> Result>> { - let next = AtomicUsize::new(0); - let mut done = std::thread::scope(|scope| { - let handles = (0..workers) - .map(|index| { - std::thread::Builder::new() - .name(format!("diffy-semantic-{index}")) - .stack_size(WORKER_STACK_BYTES) - .spawn_scoped(scope, || { - let mut done = Vec::new(); - loop { - let at = next.fetch_add(1, Ordering::Relaxed); - let Some(item) = changed.get(at) else { - return done; - }; - let _work = crate::platform::energy::work( - crate::platform::energy::Kind::Semantic, - ) - .detail(item.display_path().to_string()); - done.push((at, load(item))); - } - }) - .map_err(|error| { - DiffyError::General(format!("semantic worker setup failed: {error}")) - }) - }) - .collect::>>()?; - let mut done = Vec::with_capacity(changed.len()); - for handle in handles { - let part = handle - .join() - .map_err(|_| DiffyError::General("semantic worker panicked".to_owned()))?; - done.extend(part); - } - Ok::<_, DiffyError>(done) - })?; - done.sort_unstable_by_key(|(at, _)| *at); - done.into_iter().map(|(_, doc)| doc).collect() -} - -pub fn document_from_changed(changed: &ChangedPath) -> Result { - document_from_changed_vetted(changed, None) +/// One file's semantic diff, on the calling thread. +pub fn document_from_changed(changed: &FileInput) -> Result { + let store = Grammars::shared().store(); + let diff = diffy_compare::diff_file(changed, &Options::new(&store))?; + log_fallback(changed, &diff); + Ok(diff.document) } -fn document_from_changed_vetted( - changed: &ChangedPath, - vet: Option<&diffy_semantic::Vet<'_>>, -) -> Result { - let store = Grammars::shared().store(); - let (doc, fallback) = - document_from_native_with(changed, |tag| semantic_grammar(&store, tag), vet)?; - if let Some(reason) = fallback { +fn log_fallback(changed: &FileInput, diff: &FileDiff) { + if let Some(reason) = &diff.fallback { tracing::info!( target: "diffy::semantic", path = %changed.display_path(), - reason, - old_bytes = changed.old_content.len(), - new_bytes = changed.new_content.len(), + reason = %reason, + old_bytes = changed.old.len(), + new_bytes = changed.new.len(), "semantic diff fell back to line diff" ); } - Ok(doc) } #[cfg(feature = "difftastic")] -pub fn document_from_difftastic(changed: &ChangedPath) -> Result { - let meta = changed.meta(); - if is_binary_bytes(&changed.old_content) || is_binary_bytes(&changed.new_content) { - return binary_document(meta); - } - let old = TextStore::from_bytes(&changed.old_content); - let new = TextStore::from_bytes(&changed.new_content); - if changed.status == FileStatus::Added || changed.old_path.is_none() { - return whole_file(meta, old, new, Side::New); - } - if changed.status == FileStatus::Deleted || changed.new_path.is_none() { - return whole_file(meta, old, new, Side::Old); - } - if changed.old_content == changed.new_content { - return build(DocumentBuilder::new(), meta, old, new, |_| {}); - } +pub fn document_from_difftastic(changed: &FileInput) -> Result { + let (meta, old, new) = match diffy_compare::prepare(changed)? { + diffy_compare::Prepared::Done(doc) => return Ok(*doc), + diffy_compare::Prepared::Text { meta, old, new } => (meta, old, new), + }; let result = vendored_difftastic::diff_bytes_semantic(DiffRequest { display_path: changed.display_path(), lhs_path: changed.old_path.as_deref().map(std::path::Path::new), @@ -345,66 +270,14 @@ pub fn document_from_difftastic(changed: &ChangedPath) -> Result { }) .map_err(|error| DiffyError::General(format!("difftastic failed: {error}")))?; log_line_fallback(changed, &result); - match result.status { - DiffStatus::Binary => binary_document(meta), + let doc = match result.status { + DiffStatus::Binary => diffy_compare::binary_document(meta)?, DiffStatus::Changed if !result.chunks.is_empty() => { - document_from_semantic(meta, old, new, &result) + document_from_semantic(meta, old, new, &result)? } - _ => document_from_bytes(meta, &changed.old_content, &changed.new_content), - } -} - -fn whole_file(meta: FileMeta, old: TextStore, new: TextStore, side: Side) -> Result { - let lines = match side { - Side::Old => old.line_count(), - Side::New => new.line_count(), + _ => diffy_compare::line_diff(meta, &changed.old, &changed.new)?, }; - let (old, new) = match side { - Side::Old => (old, TextStore::default()), - Side::New => (TextStore::default(), new), - }; - build(DocumentBuilder::new(), meta, old, new, |file| { - if lines > 0 { - match side { - Side::Old => file.hunk(0, 0).change(lines, 0), - Side::New => file.hunk(0, 0).change(0, lines), - }; - } - }) -} - -fn build( - mut builder: DocumentBuilder, - meta: FileMeta, - old: TextStore, - new: TextStore, - shape: impl FnOnce(&mut diffy_diff::FileBuilder<'_>), -) -> Result { - let mut file = builder.file(meta, SourceCoverage::Full, old, new); - shape(&mut file); - file.finish() - .map_err(|error| DiffyError::Parse(error.to_string()))?; - Ok(builder.build()) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Kind { - Context, - Change, -} - -#[derive(Clone, Copy)] -struct Segment { - kind: Kind, - old: u32, - new: u32, - shown: bool, -} - -struct HunkRange { - old: std::ops::Range, - new: std::ops::Range, - segments: std::ops::Range, + Ok(doc) } #[cfg(feature = "difftastic")] @@ -417,277 +290,15 @@ fn document_from_semantic( let chunks = sanitize(result, &old, &new); let structural = result.line_fallback_reason.is_none() && line_fallback_reason(&result.language).is_none(); - document_from_rows(meta, old, new, chunks, structural, Default::default()) -} - -pub fn document_from_native( - changed: &ChangedPath, - grammars: &GrammarStore, -) -> Result<(DiffDocument, Option)> { - document_from_native_with(changed, |tag| semantic_grammar(grammars, tag), None) -} - -pub struct SemanticGrammar { - pub name: Arc, - pub language: diffy_semantic::Language, - pub rules: Arc, - pub injector: Option, -} - -pub fn semantic_tag(path: &str) -> Option { - let id = LanguageId::from_path(path)?; - match id.as_str() { - "js" | "mjs" | "cjs" => LanguageId::from_fence("typescript"), - "jsx" => LanguageId::from_fence("tsx"), - _ => Some(id), - } -} - -pub fn semantic_grammar(store: &GrammarStore, tag: &LanguageId) -> Option { - static RULES: Mutex, Arc)>> = Mutex::new(Vec::new()); - let (language, queries) = store.parser(tag.as_str())?; - let cached = |rules: &[(Arc, Arc)]| { - rules - .iter() - .find(|(highlights, _)| Arc::ptr_eq(highlights, &queries.highlights)) - .map(|(_, rules)| rules.clone()) - }; - let found = cached(&RULES.lock().unwrap_or_else(PoisonError::into_inner)); - let rules = match found { - Some(rules) => rules, - None => { - let rules = Arc::new(compile_rules(&language, &queries)); - let mut all = RULES.lock().unwrap_or_else(PoisonError::into_inner); - match cached(&all) { - Some(rules) => rules, - None => { - all.push((queries.highlights.clone(), rules.clone())); - rules - } - } - } - }; - Some(SemanticGrammar { - injector: store.injector(tag.as_str()), - name: queries.language, - language, - rules, - }) -} - -fn compile_rules(language: &diffy_semantic::Language, grammar: &GrammarQueries) -> Rules { - let queries = Queries { - highlights: &grammar.highlights, - locals: grammar.locals.as_deref(), - }; - let own = [ - diffy_semantic::semantic_query(&grammar.language), - grammar.semantic.as_deref(), - ]; - for semantic in own.into_iter().flatten() { - match Rules::new(language, semantic, &queries) { - Ok(rules) => return rules, - Err(error) => tracing::warn!( - target: "diffy::semantic", - language = %grammar.language, - %error, - "semantic query does not compile" - ), - } - } - Rules::derived(language, &queries) -} - -pub fn document_from_native_with( - changed: &ChangedPath, - grammars: impl Fn(&LanguageId) -> Option, - vet: Option<&diffy_semantic::Vet<'_>>, -) -> Result<(DiffDocument, Option)> { - let meta = changed.meta(); - if is_binary_bytes(&changed.old_content) || is_binary_bytes(&changed.new_content) { - return Ok((binary_document(meta)?, None)); - } - let old = TextStore::from_bytes(&changed.old_content); - let new = TextStore::from_bytes(&changed.new_content); - if changed.status == FileStatus::Added || changed.old_path.is_none() { - return Ok((whole_file(meta, old, new, Side::New)?, None)); - } - if changed.status == FileStatus::Deleted || changed.new_path.is_none() { - return Ok((whole_file(meta, old, new, Side::Old)?, None)); - } - if changed.old_content == changed.new_content { - return Ok((build(DocumentBuilder::new(), meta, old, new, |_| {})?, None)); - } - let line_diff = |reason: String| { - document_from_bytes(meta.clone(), &changed.old_content, &changed.new_content) - .map(|doc| (doc, Some(reason))) - }; - let Some(tag) = semantic_tag(changed.display_path()) else { - return line_diff("no grammar for this file".to_owned()); - }; - let Some(grammar) = grammars(&tag) else { - return line_diff(format!("{tag} grammar unavailable")); - }; - let limits = Limits { - max_time: FILE_TIME_BUDGET, - ..Limits::default() - }; - let embed = |tree: &diffy_semantic::Tree, source: &str| { - let Some(injector) = &grammar.injector else { - return Vec::new(); - }; - // Regions repeat a few languages many times (every `{expression}` - // of a Svelte template), so each is looked up once. - let mut seen: Vec<(LanguageId, Option)> = Vec::new(); - let mut out = Vec::new(); - for (language, range) in injector.find(tree, source) { - let at = match seen.iter().position(|(l, _)| *l == language) { - Some(at) => at, - None => { - let found = grammars(&language); - seen.push((language, found)); - seen.len() - 1 - } - }; - if let Some(embedded) = &seen[at].1 { - out.push(diffy_semantic::Embedded { - range, - language: embedded.language.clone(), - rules: embedded.rules.clone(), - name: embedded.name.clone(), - }); - } - } - out - }; - let diff = diffy_semantic::diff_with( - &grammar.language, - &grammar.rules, - old.as_str(), - new.as_str(), - &limits, - vet, - Some(&embed), - ); - match diff { - Ok(diff) => { - let chunks = clean_rows(diff.chunks, &old, &new); - let groups = diff - .groups - .into_iter() - .map(|g| diffy_diff::LineGroupSpan { - old: g.old, - new: g.new, - }) - .collect(); - let semantics = (diff.outline, diff.renames, diff.moves, groups); - let doc = document_from_rows(meta, old, new, chunks, true, semantics)?; - Ok((doc, None)) - } - Err(fallback) => line_diff(fallback.to_string()), - } -} - -fn clean_rows(chunks: Vec>, old: &TextStore, new: &TextStore) -> Vec> { - let side = |store: &TextStore, line: Option, spans: Vec| { - let line = line.filter(|&l| l < store.line_count()); - let spans = line - .and_then(|l| store.display_line(l)) - .map(|text| clean_spans(spans, text)) - .unwrap_or_default(); - (line, spans) - }; - chunks - .into_iter() - .map(|chunk| { - chunk - .into_iter() - .map(|row| { - let [old_spans, new_spans] = row.spans; - let (old_line, old_spans) = side(old, row.old, old_spans); - let (new_line, new_spans) = side(new, row.new, new_spans); - Row { - old: old_line, - new: new_line, - spans: [old_spans, new_spans], - } - }) - .collect() - }) - .collect() -} - -fn document_from_rows( - meta: FileMeta, - old: TextStore, - new: TextStore, - chunks: Vec>, - structural: bool, - (outline, renames, moves, groups): ( - Vec, - Vec, - Vec, - Vec, - ), -) -> Result { - let segments = canonical_segments(&chunks, &old, &new); - let hunks = hunk_ranges(&segments); - let mut rows: Vec> = hunks.iter().map(|_| Vec::new()).collect(); - let mut spans: [Vec<(u32, Vec)>; 2] = [Vec::new(), Vec::new()]; - let mut seen = [ - std::collections::HashSet::new(), - std::collections::HashSet::new(), - ]; - for row in chunks.iter().flatten() { - let old_hunk = row.old.and_then(|line| find_hunk(&hunks, Side::Old, line)); - let new_hunk = row.new.and_then(|line| find_hunk(&hunks, Side::New, line)); - if let Some(h) = old_hunk.or(new_hunk) { - let range = &hunks[h]; - let aligned = AlignedRow { - old: row.old.filter(|line| range.old.contains(line)), - new: row.new.filter(|line| range.new.contains(line)), - }; - if aligned.old.is_some() || aligned.new.is_some() { - rows[h].push(aligned); - } - } - for (side, line, hunk) in [ - (Side::Old, row.old, old_hunk), - (Side::New, row.new, new_hunk), - ] { - let s = side as usize; - if let (Some(line), Some(_)) = (line, hunk) - && !row.spans[s].is_empty() - && seen[s].insert(line) - { - spans[s].push((line, row.spans[s].clone())); - } - } - } - build(DocumentBuilder::new(), meta, old, new, |file| { - file.structural(structural); - file.outline(outline); - file.renames(renames); - file.moves(moves); - file.groups(groups); - for (hunk, hunk_rows) in hunks.iter().zip(rows) { - file.hunk(hunk.old.start, hunk.new.start); - for segment in &segments[hunk.segments.clone()] { - match segment.kind { - Kind::Context => file.context(segment.old), - Kind::Change => file.change(segment.old, segment.new), - }; - } - for row in hunk_rows { - file.row(row); - } - } - for side in [Side::Old, Side::New] { - for (line, line_spans) in &spans[side as usize] { - file.spans(side, *line, line_spans); - } - } - }) + let doc = diffy_compare::document_from_rows( + meta, + old, + new, + chunks, + structural, + diffy_compare::Semantics::default(), + )?; + Ok(doc) } #[cfg(feature = "difftastic")] @@ -731,34 +342,7 @@ fn sanitize_spans(spans: &[ChangeSpan], text: &str) -> Vec { intensity: intensity(span.intensity), }) .collect(); - clean_spans(spans, text) -} - -fn clean_spans(spans: Vec, text: &str) -> Vec { - let mut out: Vec = spans - .into_iter() - .filter_map(|span| { - let start = span.start as usize; - let end = (span.end as usize).min(text.len()); - (start < end && text.is_char_boundary(start) && text.is_char_boundary(end)).then_some( - SemanticSpan { - start: start as u32, - end: end as u32, - intensity: span.intensity, - }, - ) - }) - .collect(); - out.sort_by_key(|span| (span.start, span.end)); - let mut end = 0; - out.retain(|span| { - let keep = span.start >= end; - if keep { - end = span.end; - } - keep - }); - out + diffy_compare::clean_spans(spans, text) } #[cfg(feature = "difftastic")] @@ -770,177 +354,6 @@ fn intensity(intensity: DftIntensity) -> ChangeIntensity { } } -fn is_context_row(row: &Row, old: &TextStore, new: &TextStore, lo: u32, ln: u32) -> bool { - old.line(lo) == new.line(ln) - && row - .spans - .iter() - .flatten() - .all(|span| span.intensity == ChangeIntensity::UnchangedContext) -} - -fn canonical_segments(chunks: &[Vec], old: &TextStore, new: &TextStore) -> Vec { - let mut segments = Vec::new(); - let (mut o, mut n) = (0u32, 0u32); - for chunk in chunks { - let mut first = true; - for row in chunk { - let lo = row.old.filter(|&line| line >= o); - let ln = row.new.filter(|&line| line >= n); - if lo.is_none() && ln.is_none() { - continue; - } - let (to, tn) = (lo.unwrap_or(o), ln.unwrap_or(n)); - push_unmentioned(&mut segments, old, new, o..to, n..tn, !first); - first = false; - let kind = match (lo, ln) { - (Some(a), Some(b)) if is_context_row(row, old, new, a, b) => Kind::Context, - _ => Kind::Change, - }; - segments.push(Segment { - kind, - old: u32::from(lo.is_some()), - new: u32::from(ln.is_some()), - shown: true, - }); - o = to + u32::from(lo.is_some()); - n = tn + u32::from(ln.is_some()); - } - } - push_unmentioned( - &mut segments, - old, - new, - o..old.line_count(), - n..new.line_count(), - false, - ); - segments -} - -fn push_unmentioned( - segments: &mut Vec, - old: &TextStore, - new: &TextStore, - old_lines: std::ops::Range, - new_lines: std::ops::Range, - inside: bool, -) { - let (old_len, new_len) = (old_lines.len() as u32, new_lines.len() as u32); - if old_len == 0 && new_len == 0 { - return; - } - let equal = old_len == new_len - && old_lines - .clone() - .zip(new_lines.clone()) - .all(|(a, b)| old.line(a) == new.line(b)); - if equal { - segments.push(Segment { - kind: Kind::Context, - old: old_len, - new: new_len, - shown: inside, - }); - return; - } - let changes = line_changes( - lines_bytes(old, old_lines.clone()), - lines_bytes(new, new_lines.clone()), - ); - let mut pieces = Vec::with_capacity(changes.len() * 2 + 1); - let (mut o, mut n) = (0u32, 0u32); - for change in &changes { - if change.old.start > o { - pieces.push((Kind::Context, change.old.start - o, change.new.start - n)); - } - pieces.push(( - Kind::Change, - change.old.len() as u32, - change.new.len() as u32, - )); - o = change.old.end; - n = change.new.end; - } - if o < old_len { - pieces.push((Kind::Context, old_len - o, new_len - n)); - } - let last = pieces.len().saturating_sub(1); - for (index, (kind, old, new)) in pieces.into_iter().enumerate() { - let edge = index == 0 || index == last; - segments.push(Segment { - kind, - old, - new, - shown: kind == Kind::Change || !edge || inside, - }); - } -} - -fn lines_bytes(store: &TextStore, lines: std::ops::Range) -> &[u8] { - let text = store.as_str(); - let offset = |line: u32| { - store - .line_range(line) - .map_or(text.len(), |range| range.start) - }; - &text.as_bytes()[offset(lines.start)..offset(lines.end)] -} - -fn hunk_ranges(segments: &[Segment]) -> Vec { - let mut hunks = Vec::new(); - let (mut o, mut n) = (0u32, 0u32); - let mut open: Option<(u32, u32, usize, bool)> = None; - for (index, segment) in segments.iter().enumerate() { - let in_hunk = segment.shown || segment.kind == Kind::Change; - match (&mut open, in_hunk) { - (None, true) => open = Some((o, n, index, segment.kind == Kind::Change)), - (Some(state), true) => state.3 |= segment.kind == Kind::Change, - (Some(_), false) => close(&mut hunks, open.take(), o, n, index), - (None, false) => {} - } - o += segment.old; - n += segment.new; - } - close(&mut hunks, open, o, n, segments.len()); - hunks -} - -fn close( - hunks: &mut Vec, - open: Option<(u32, u32, usize, bool)>, - o: u32, - n: u32, - end: usize, -) { - if let Some((start_o, start_n, start, true)) = open { - hunks.push(HunkRange { - old: start_o..o, - new: start_n..n, - segments: start..end, - }); - } -} - -fn find_hunk(hunks: &[HunkRange], side: Side, line: u32) -> Option { - let range = |h: &HunkRange| match side { - Side::Old => h.old.clone(), - Side::New => h.new.clone(), - }; - let index = hunks.partition_point(|h| range(h).end <= line); - (index < hunks.len() && range(&hunks[index]).contains(&line)).then_some(index) -} - -fn worker_count(files: usize) -> usize { - if files < MIN_FILES_FOR_PARALLEL { - return 1; - } - let available = std::thread::available_parallelism() - .map(usize::from) - .unwrap_or(1); - files.min(available).clamp(1, MAX_WORKERS) -} - #[cfg(feature = "difftastic")] fn line_fallback_reason(language: &str) -> Option<&str> { language @@ -949,7 +362,7 @@ fn line_fallback_reason(language: &str) -> Option<&str> { } #[cfg(feature = "difftastic")] -fn log_line_fallback(changed: &ChangedPath, result: &SemanticDiffResult) { +fn log_line_fallback(changed: &FileInput, result: &SemanticDiffResult) { if let Some(reason) = result .line_fallback_reason .as_deref() @@ -960,8 +373,8 @@ fn log_line_fallback(changed: &ChangedPath, result: &SemanticDiffResult) { path = %changed.display_path(), reason, chunks = result.chunks.len(), - old_bytes = changed.old_content.len(), - new_bytes = changed.new_content.len(), + old_bytes = changed.old.len(), + new_bytes = changed.new.len(), "difftastic semantic diff fell back to line diff" ); } @@ -972,7 +385,7 @@ fn read_changed_paths( left: &str, right: &str, entries: Vec<(String, Option, Option)>, -) -> Vec { +) -> Vec { let old_paths = entries.iter().filter_map(|(_, old, _)| old.as_deref()); let mut old_contents = git.read_file_bytes_batch_at(left, old_paths).into_iter(); let new_paths = entries.iter().filter_map(|(_, _, new)| new.as_deref()); @@ -986,12 +399,12 @@ fn read_changed_paths( let new_content = new_path .as_ref() .and_then(|_| new_contents.next().flatten()); - ChangedPath { + FileInput { status: status_from_label(&status), old_path, new_path, - old_content: old_content.unwrap_or_default(), - new_content: new_content.unwrap_or_default(), + old: old_content.unwrap_or_default(), + new: new_content.unwrap_or_default(), } }) .collect() @@ -999,190 +412,13 @@ fn read_changed_paths( #[cfg(test)] mod tests { - use diffy_diff::{ - BlockKind, ChangeIntensity as Intensity, DiffDocument, FileMeta, FileStatus, SemanticSpan, - Side, TextStore, - }; - use diffy_semantic::Row; + use diffy_diff::{FileStatus, Side}; - use super::{ - ChangedPath, clean_rows, compare, compare_path, document_from_changed, document_from_rows, - }; + use super::{compare, compare_path}; use crate::core::compare::service::tests::commit_file; use crate::core::compare::spec::{CompareMode, CompareSpec, LayoutMode, RendererKind}; use crate::core::vcs::git::{GitService, WORKDIR_REF}; - fn span(start: u32, end: u32, intensity: Intensity) -> SemanticSpan { - SemanticSpan { - start, - end, - intensity, - } - } - - fn line( - old: Option, - new: Option, - old_spans: Vec, - new_spans: Vec, - ) -> Row { - Row { - old, - new, - spans: [old_spans, new_spans], - } - } - - fn rows_document(old: &str, new: &str, chunks: Vec>) -> DiffDocument { - let (old, new) = (TextStore::new(old), TextStore::new(new)); - let chunks = clean_rows(chunks, &old, &new); - document_from_rows(meta(), old, new, chunks, true, Default::default()).unwrap() - } - - fn meta() -> FileMeta { - FileMeta { - old_path: Some("src/lib.rs".into()), - new_path: Some("src/lib.rs".into()), - status: FileStatus::Modified, - ..FileMeta::default() - } - } - - fn hunks(doc: &diffy_diff::DiffDocument) -> Vec<(u32, u32, u32, u32)> { - let h = doc.hunks(); - (0..doc.hunk_count() as usize) - .map(|i| (h.old_start[i], h.old_len[i], h.new_start[i], h.new_len[i])) - .collect() - } - - #[test] - fn semantic_rows_keep_intensities_pairing_and_full_sources() { - let old = "fn a() {}\nlet x = \"foo\";\nkeep();\n"; - let new = "fn a() {}\nlet x = \"bar\";\nextra();\nkeep();\n"; - let doc = rows_document( - old, - new, - vec![vec![ - line(Some(0), Some(0), vec![], vec![]), - line( - Some(1), - Some(1), - vec![ - span(8, 9, Intensity::UnchangedContext), - span(9, 12, Intensity::NovelWord), - span(12, 13, Intensity::UnchangedContext), - ], - vec![span(9, 12, Intensity::NovelWord)], - ), - line(None, Some(2), vec![], vec![span(0, 7, Intensity::Novel)]), - line(Some(2), Some(3), vec![], vec![]), - ]], - ); - doc.verify_integrity().unwrap(); - assert_eq!(hunks(&doc), [(1, 3, 1, 4)]); - assert_eq!( - doc.blocks().kind, - [BlockKind::Context, BlockKind::Change, BlockKind::Context] - ); - assert_eq!(doc.stats(0).known(), Some((2, 1))); - assert!(doc.prefers_structural(0)); - let intensities: Vec<_> = doc - .semantic_spans(0, Side::Old, 1) - .iter() - .map(|s| s.intensity) - .collect(); - assert_eq!( - intensities, - [ - Intensity::UnchangedContext, - Intensity::NovelWord, - Intensity::UnchangedContext - ] - ); - assert_eq!( - doc.semantic_spans(0, Side::New, 2)[0].intensity, - Intensity::Novel - ); - let rows: Vec<_> = doc.aligned_rows(0).iter().map(|r| (r.old, r.new)).collect(); - assert_eq!( - rows, - [ - (Some(0), Some(0)), - (Some(1), Some(1)), - (None, Some(2)), - (Some(2), Some(3)) - ] - ); - assert_eq!(doc.text(0, Side::New).line_count(), 4); - } - - #[test] - fn lines_that_differ_between_chunks_become_changes() { - let old = "a\nb\nsame\nc\nd\n"; - let new = "A\nb\nsame2\nc\nD\n"; - let doc = rows_document( - old, - new, - vec![ - vec![line( - Some(0), - Some(0), - vec![span(0, 1, Intensity::Novel)], - vec![span(0, 1, Intensity::Novel)], - )], - vec![line( - Some(4), - Some(4), - vec![span(0, 1, Intensity::Novel)], - vec![span(0, 1, Intensity::Novel)], - )], - ], - ); - doc.verify_integrity().unwrap(); - assert_eq!(hunks(&doc), [(1, 1, 1, 1), (3, 1, 3, 1), (5, 1, 5, 1)]); - assert_eq!(doc.stats(0).known(), Some((3, 3))); - } - - #[test] - fn separate_chunks_stay_separate_hunks_and_bad_spans_are_dropped() { - let old = "a\n1\n2\n3\n4\nb\n"; - let new = "x\n1\n2\n3\n4\ny\n"; - let doc = rows_document( - old, - new, - vec![ - vec![line( - Some(0), - Some(0), - vec![span(0, 9, Intensity::Novel)], - vec![], - )], - vec![ - line(Some(5), Some(5), vec![], vec![span(3, 2, Intensity::Novel)]), - line(Some(6), Some(6), vec![], vec![]), - ], - ], - ); - assert_eq!(hunks(&doc), [(1, 1, 1, 1), (6, 1, 6, 1)]); - assert_eq!(doc.semantic_spans(0, Side::Old, 0)[0].end, 1); - assert!(doc.semantic_spans(0, Side::New, 5).is_empty()); - } - - #[test] - fn added_files_skip_the_semantic_diff() { - let doc = document_from_changed(&ChangedPath { - status: FileStatus::Added, - old_path: None, - new_path: Some("src/new.rs".to_owned()), - old_content: Vec::new(), - new_content: b"fn new() {}\nlet x = 1;\n".to_vec(), - }) - .unwrap(); - assert_eq!(hunks(&doc), [(0, 0, 1, 2)]); - assert_eq!(doc.text(0, Side::New).line(1), Some("let x = 1;")); - assert_eq!(doc.summary(0).status, FileStatus::Added); - } - fn repo_spec(left: &str, right: &str) -> CompareSpec { CompareSpec { mode: CompareMode::TwoDot, diff --git a/src/core/error.rs b/src/core/error.rs index e4421ec8..3863c19e 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -140,4 +140,14 @@ impl DiffyError { } } +impl From for DiffyError { + fn from(error: diffy_compare::Error) -> Self { + match error { + // Keeps the message document builds have always surfaced. + diffy_compare::Error::Build(error) => Self::Parse(error), + error => Self::General(error.to_string()), + } + } +} + pub type Result = std::result::Result; diff --git a/src/core/vcs/jj/service.rs b/src/core/vcs/jj/service.rs index 6dd5fd7e..8d3b9aa7 100644 --- a/src/core/vcs/jj/service.rs +++ b/src/core/vcs/jj/service.rs @@ -191,7 +191,7 @@ impl JjRepository { change: &FileChange, old_rev: &str, new_rev: &str, - ) -> Result { + ) -> Result { use crate::core::vcs::compare_port::file_status; use crate::core::vcs::model::FileChangeStatus; let old_path = match change.status { @@ -212,10 +212,10 @@ impl JjRepository { .transpose() .map(Option::unwrap_or_default) }; - Ok(semantic::ChangedPath { + Ok(diffy_compare::FileInput { status: file_status(change.status), - old_content: read(old_rev, old_path)?, - new_content: read(new_rev, new_path)?, + old: read(old_rev, old_path)?, + new: read(new_rev, new_path)?, old_path: old_path.map(str::to_owned), new_path: new_path.map(str::to_owned), }) diff --git a/tools/semantic-eval/Cargo.toml b/tools/semantic-eval/Cargo.toml index 5861833f..0c92bae8 100644 --- a/tools/semantic-eval/Cargo.toml +++ b/tools/semantic-eval/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] clap = { version = "4.5", default-features = false, features = ["derive", "error-context", "help", "std", "usage"] } diffy = { path = "../..", features = ["difftastic"] } +diffy-compare = { path = "../../crates/diffy-compare" } diffy-diff = { path = "../../crates/diffy-diff" } diffy-syntax = { path = "../../crates/diffy-syntax", features = ["download"] } diffy-semantic = { path = "../../crates/diffy-semantic", features = ["eval"] } diff --git a/tools/semantic-eval/src/engine.rs b/tools/semantic-eval/src/engine.rs index 12d32236..aca7869b 100644 --- a/tools/semantic-eval/src/engine.rs +++ b/tools/semantic-eval/src/engine.rs @@ -1,11 +1,9 @@ use std::sync::OnceLock; use std::time::Instant; -use diffy::core::compare::semantic::{ - ChangedPath, SemanticGrammar, document_from_difftastic, document_from_native_with, - semantic_grammar, semantic_tag, -}; +use diffy::core::compare::semantic::document_from_difftastic; use diffy::core::syntax::grammar::GrammarConfig; +use diffy_compare::{FileInput, Options, SemanticGrammar, semantic_grammar, semantic_tag}; use diffy_diff::{DEFAULT_CONTEXT, DiffDocument, FileStatus, diff_texts}; use diffy_syntax::{GrammarStore, LanguageId, LanguageStatus}; use serde::{Deserialize, Serialize}; @@ -37,23 +35,23 @@ pub struct Run { } pub fn run(engine: Engine, path: &str, old: &[u8], new: &[u8]) -> Result { - let changed = || ChangedPath { + let changed = || FileInput { status: FileStatus::Modified, old_path: Some(path.to_owned()), new_path: Some(path.to_owned()), - old_content: old.to_vec(), - new_content: new.to_vec(), + old: old.to_vec(), + new: new.to_vec(), }; match engine { Engine::Native => { let changed = changed(); let start = Instant::now(); - let (doc, fallback) = document_from_native_with(&changed, tagged, None) + let diff = diffy_compare::diff_file_with(&changed, &Options::new(grammars()), &tagged) .map_err(|error| error.to_string())?; Ok(Run { - doc, + doc: diff.document, micros: start.elapsed().as_micros() as u64, - fallback, + fallback: diff.fallback.map(|fallback| fallback.to_string()), }) } Engine::Difftastic => { From 74034226a2792be749fa4fb54e6d56343c888400 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:39:04 +0000 Subject: [PATCH 06/16] feat(node): add the @diffy/node binding A napi-rs binding over diffy-compare for Node servers and Electron: diff() and expand() run each request on its own thread (never libuv's pool), settle a promise through a thread-safe function, and return the versioned display format as JSON. An AbortSignal cancels a request and a request budget falls back to line diffs. Grammars are compiled in from a syntax-pack bundle. The workspace gains a `node` profile (release with panic = "unwind") so a Rust panic rejects the call instead of aborting the host process. The app binary is unchanged. CI builds darwin-arm64, darwin-x64 (cross), linux-x64-gnu and linux-arm64-gnu (inside manylinux_2_28, glibc <= 2.28 checked) and win32-x64-msvc, tests each natively, and publishes on node-v* tags with npm provenance. --- .github/workflows/node.yml | 254 ++++++++++++++++ AGENTS.md | 2 + Cargo.lock | 147 +++++++++- Cargo.toml | 7 + bindings/node/.gitignore | 5 + bindings/node/Cargo.toml | 32 ++ bindings/node/LICENSE-APACHE | 202 +++++++++++++ bindings/node/LICENSE-MIT | 21 ++ bindings/node/README.md | 108 +++++++ bindings/node/binding.js | 41 +++ bindings/node/build.rs | 3 + bindings/node/index.d.ts | 274 ++++++++++++++++++ bindings/node/index.js | 72 +++++ bindings/node/npm/darwin-arm64/README.md | 5 + bindings/node/npm/darwin-arm64/package.json | 31 ++ bindings/node/npm/darwin-x64/README.md | 5 + bindings/node/npm/darwin-x64/package.json | 31 ++ bindings/node/npm/linux-arm64-gnu/README.md | 5 + .../node/npm/linux-arm64-gnu/package.json | 34 +++ bindings/node/npm/linux-x64-gnu/README.md | 5 + bindings/node/npm/linux-x64-gnu/package.json | 34 +++ bindings/node/npm/win32-x64-msvc/README.md | 5 + bindings/node/npm/win32-x64-msvc/package.json | 31 ++ bindings/node/package.json | 47 +++ bindings/node/scripts/build.mjs | 75 +++++ bindings/node/src/lib.rs | 273 +++++++++++++++++ bindings/node/test/diffy.test.mjs | 251 ++++++++++++++++ 27 files changed, 1991 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/node.yml create mode 100644 bindings/node/.gitignore create mode 100644 bindings/node/Cargo.toml create mode 100644 bindings/node/LICENSE-APACHE create mode 100644 bindings/node/LICENSE-MIT create mode 100644 bindings/node/README.md create mode 100644 bindings/node/binding.js create mode 100644 bindings/node/build.rs create mode 100644 bindings/node/index.d.ts create mode 100644 bindings/node/index.js create mode 100644 bindings/node/npm/darwin-arm64/README.md create mode 100644 bindings/node/npm/darwin-arm64/package.json create mode 100644 bindings/node/npm/darwin-x64/README.md create mode 100644 bindings/node/npm/darwin-x64/package.json create mode 100644 bindings/node/npm/linux-arm64-gnu/README.md create mode 100644 bindings/node/npm/linux-arm64-gnu/package.json create mode 100644 bindings/node/npm/linux-x64-gnu/README.md create mode 100644 bindings/node/npm/linux-x64-gnu/package.json create mode 100644 bindings/node/npm/win32-x64-msvc/README.md create mode 100644 bindings/node/npm/win32-x64-msvc/package.json create mode 100644 bindings/node/package.json create mode 100644 bindings/node/scripts/build.mjs create mode 100644 bindings/node/src/lib.rs create mode 100644 bindings/node/test/diffy.test.mjs diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml new file mode 100644 index 00000000..e234dcdd --- /dev/null +++ b/.github/workflows/node.yml @@ -0,0 +1,254 @@ +name: Node binding + +# Builds @diffy/node for every platform it ships, tests each on a native +# runner, and publishes the packages on a `node-v` tag. + +on: + push: + tags: ["node-v*"] + pull_request: + paths: + - "bindings/node/**" + - "crates/diffy-compare/**" + - "crates/diffy-diff/**" + - "crates/diffy-semantic/**" + - "crates/diffy-syntax/**" + - "tools/syntax-pack/**" + - "Cargo.lock" + - "Cargo.toml" + - "rust-toolchain.toml" + - ".github/workflows/node.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: node-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} + +env: + TREE_SITTER_CLI: 0.26.8 + NODE_VERSION: "22" + # Node 22 and Electron need macOS 11; C grammars and Rust agree on it. + MACOSX_DEPLOYMENT_TARGET: "11.0" + # The oldest glibc the Linux binaries may need (manylinux_2_28). + MAX_GLIBC: "2.28" + +jobs: + build: + name: build ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + target: aarch64-apple-darwin + os: macos-26 + # Cross-built on Apple silicon; tested with an x64 Node under Rosetta. + - platform: darwin-x64 + target: x86_64-apple-darwin + os: macos-26 + node-arch: x64 + # Built and tested inside manylinux_2_28 (glibc 2.28), so the + # grammars' C and the Rust link both see that glibc and nothing newer. + - platform: linux-x64-gnu + target: x86_64-unknown-linux-gnu + os: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64 + - platform: linux-arm64-gnu + target: aarch64-unknown-linux-gnu + os: ubuntu-24.04-arm + container: quay.io/pypa/manylinux_2_28_aarch64 + - platform: win32-x64-msvc + target: x86_64-pc-windows-msvc + os: windows-2025 + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + timeout-minutes: 120 + defaults: + run: + shell: bash + env: + PLATFORM: ${{ matrix.platform }} + TARGET: ${{ matrix.target }} + steps: + # Before checkout: it needs git for the submodule. The image's GCC + # toolset compiles the grammars and links Rust (in place of the clang + # .cargo/config.toml names, keeping its -fuse-ld=lld), so both build + # against the image's glibc. + - name: Install build tools (manylinux) + if: matrix.container + run: | + dnf install -y git lld + echo "CC=gcc" >> "$GITHUB_ENV" + echo "CARGO_TARGET_$(echo "$TARGET" | tr 'a-z-' 'A-Z_')_LINKER=gcc" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install rustup (manylinux) + if: matrix.container + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Install the pinned Rust toolchain + run: | + rustup toolchain install + rustup target add "$TARGET" + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: node-${{ matrix.target }}-v1 + save-if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }} + + - name: Install lld (macOS) + if: runner.os == 'macOS' + run: brew install lld + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + architecture: ${{ matrix.node-arch }} + + - name: Install tree-sitter CLI (npm) + if: ${{ !matrix.container }} + run: | + npm install --global "tree-sitter-cli@${TREE_SITTER_CLI}" + cli="$(npm root -g)/tree-sitter-cli/tree-sitter" + if [ "${{ runner.os }}" = Windows ]; then cli="$cli.exe"; fi + "$cli" --version + echo "TREE_SITTER=$cli" >> "$GITHUB_ENV" + + # npm's prebuilt CLI needs a newer glibc than the container has. + - uses: actions/cache@v4 + if: matrix.container + id: tree-sitter + with: + path: ~/tree-sitter-cli + key: tree-sitter-cli-${{ env.TREE_SITTER_CLI }}-${{ matrix.target }}-manylinux_2_28 + + - name: Install tree-sitter CLI (cargo) + if: matrix.container + run: | + if [ "${{ steps.tree-sitter.outputs.cache-hit }}" != true ]; then + cargo install --locked --root "$HOME/tree-sitter-cli" tree-sitter-cli --version "$TREE_SITTER_CLI" + fi + cli="$HOME/tree-sitter-cli/bin/tree-sitter" + "$cli" --version + echo "TREE_SITTER=$cli" >> "$GITHUB_ENV" + + - name: Bundle grammars + run: | + bundle="$GITHUB_WORKSPACE/target/syntax-bundle" + echo "DIFFY_SYNTAX_BUNDLE=$bundle" >> "$GITHUB_ENV" + cargo run --locked --release -p syntax-pack -- bundle --target "$TARGET" --out "$bundle" --jobs 4 + + - name: Build the binding + run: | + out="bindings/node/npm/$PLATFORM" + node bindings/node/scripts/build.mjs --locked --target "$TARGET" --out "$out" + cp bindings/node/LICENSE-MIT bindings/node/LICENSE-APACHE "$out/" + cp "target/syntax-bundle/$TARGET/THIRD_PARTY_NOTICES.md" "$out/" + ls -l "$out" + + - name: Sign ad hoc (macOS) + if: runner.os == 'macOS' + run: codesign --force --sign - "bindings/node/npm/$PLATFORM/diffy.$PLATFORM.node" + + - name: Check the glibc it needs (Linux) + if: runner.os == 'Linux' + run: | + node="bindings/node/npm/$PLATFORM/diffy.$PLATFORM.node" + need=$(objdump -T "$node" | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' | sort -uV | tail -1) + echo "needs glibc $need" + if [ "$(printf '%s\n%s\n' "$need" "$MAX_GLIBC" | sort -V | tail -1)" != "$MAX_GLIBC" ]; then + echo "::error::$PLATFORM needs glibc $need, newer than $MAX_GLIBC" + exit 1 + fi + + - name: Test + working-directory: bindings/node + run: | + node -p "'node ' + process.version + ' ' + process.platform + '-' + process.arch" + DIFFY_NODE_BINARY="npm/$PLATFORM/diffy.$PLATFORM.node" npm test + + - uses: actions/upload-artifact@v4 + with: + name: node-${{ matrix.platform }} + path: bindings/node/npm/${{ matrix.platform }}/ + if-no-files-found: error + + publish: + needs: build + if: startsWith(github.ref, 'refs/tags/node-v') + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + # npm provenance. + id-token: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + + - uses: actions/download-artifact@v4 + with: + pattern: node-* + path: artifacts + + - name: Assemble the platform packages + run: | + for dir in bindings/node/npm/*/; do + platform=$(basename "$dir") + cp -R "artifacts/node-$platform/." "$dir" + test -f "$dir/diffy.$platform.node" + test -f "$dir/THIRD_PARTY_NOTICES.md" + done + + - name: Check versions against the tag + env: + TAG: ${{ github.ref_name }} + run: | + node - <<'EOF' + const { readFileSync, readdirSync } = require('node:fs') + const version = process.env.TAG.replace(/^node-v/, '') + const read = (path) => JSON.parse(readFileSync(path, 'utf8')) + const main = read('bindings/node/package.json') + const problems = [] + if (main.version !== version) problems.push(`@diffy/node is ${main.version}`) + for (const platform of readdirSync('bindings/node/npm')) { + const pkg = read(`bindings/node/npm/${platform}/package.json`) + if (pkg.version !== version) problems.push(`${pkg.name} is ${pkg.version}`) + if (main.optionalDependencies[pkg.name] !== version) problems.push(`@diffy/node pins ${pkg.name}@${main.optionalDependencies[pkg.name]}`) + } + if (Object.keys(main.optionalDependencies).length !== readdirSync('bindings/node/npm').length) problems.push('optionalDependencies and npm/ differ') + if (problems.length) { + console.error(`tag ${process.env.TAG} wants ${version}:\n ${problems.join('\n ')}`) + process.exit(1) + } + EOF + + # Platform packages first, so @diffy/node never names a missing one. + # A version already on npm is skipped, so a failed run can be rerun. + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + publish() { + name=$(node -p "require('./$1/package.json').name") + version=$(node -p "require('./$1/package.json').version") + if npm view "$name@$version" version > /dev/null 2>&1; then + echo "$name@$version is already published" + else + npm publish "./$1" --provenance --access public + fi + } + for dir in bindings/node/npm/*; do publish "$dir"; done + publish bindings/node diff --git a/AGENTS.md b/AGENTS.md index 82ba3784..13f78142 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,8 @@ and syntax code stays here. `crates/diffy-syntax/` — tree-sitter packs (see its `PACKS.md`); `tools/syntax-pack/` — pack builder; `registry.toml` is generated by `syntax-pack registry`, never edited by hand. +- `bindings/node/` — `@diffy/node`, the napi-rs binding over `diffy-compare` + with bundled grammars; build it with the `node` profile (see its README). - `crates/diffy-semantic/` — structural diff behind the semantic renderer; `tools/semantic-eval/` measures it against history and fixtures. - `crates/difftastic/` is vendored and out-of-workspace, built only for diff --git a/Cargo.lock b/Cargo.lock index cdfaf1c0..b7eccd71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,7 +305,7 @@ version = "0.38.0+1.3.281" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" dependencies = [ - "libloading", + "libloading 0.8.9", ] [[package]] @@ -808,6 +808,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -994,6 +1003,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" + [[package]] name = "cursor-icon" version = "1.2.0" @@ -1118,7 +1133,7 @@ dependencies = [ "ignore", "lazy_static", "libc", - "libloading", + "libloading 0.8.9", "line-numbers", "log", "owo-colors", @@ -1262,6 +1277,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "diffy-node" +version = "0.3.7" +dependencies = [ + "diffy-compare", + "diffy-syntax", + "napi", + "napi-build", + "napi-derive", + "serde_json", +] + [[package]] name = "diffy-semantic" version = "0.3.7" @@ -1275,7 +1302,7 @@ name = "diffy-syntax" version = "0.3.7" dependencies = [ "diffy-syntax", - "libloading", + "libloading 0.8.9", "quark-update", "ring", "serde", @@ -1372,7 +1399,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" dependencies = [ - "libloading", + "libloading 0.8.9", ] [[package]] @@ -1749,6 +1776,21 @@ dependencies = [ "libc", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1756,6 +1798,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1764,6 +1807,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -1812,10 +1866,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -3521,7 +3578,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading", + "libloading 0.8.9", "pkg-config", ] @@ -3614,6 +3671,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -3895,6 +3962,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "napi" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907ff7366fb7182e28edd0f08ef8ac5d6b13b8d65a3c0b3a10bf3ac092437d6b" +dependencies = [ + "bitflags 2.11.0", + "ctor", + "futures", + "libc", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash 2.1.2", +] + +[[package]] +name = "napi-build" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b899b545d3aa6dca985939059f258c5488d34e4ecf39c274e20009748f4b846d" + +[[package]] +name = "napi-derive" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32eb00f92f84cf462fc328f49082b29c2a7d2eebeb1d447d19c1f9fdd9f89ee" +dependencies = [ + "convert_case", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "napi-derive-backend" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fe28c64d8fc12e69f9b817d0febdbe7914555760f1e353106281b8ff99cc90" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "napi-sys" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d153beb7d5c973f4b9395be8a83054a28968b61188f1dd1a2cc2321af22daea2" +dependencies = [ + "libloading 0.9.0", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3960,6 +4083,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -6137,7 +6266,7 @@ checksum = "5dbb9f2928b6654ccc28d4ddfef5213e97ed66afed4907774d049b376c62a838" dependencies = [ "js-sys", "libc", - "libloading", + "libloading 0.8.9", "memfd", "memmap2", "serde", @@ -6238,7 +6367,7 @@ version = "0.3.7" dependencies = [ "cc", "diffy-syntax", - "libloading", + "libloading 0.8.9", "quark-update", "ring", "serde", @@ -7828,7 +7957,7 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading", + "libloading 0.8.9", "log", "metal", "naga", @@ -8303,7 +8432,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "libloading", + "libloading 0.8.9", "once_cell", "rustix 1.1.4", "x11rb-protocol", diff --git a/Cargo.toml b/Cargo.toml index a2450aca..35e0a2f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ ".", + "bindings/node", "crates/diffy-compare", "crates/diffy-diff", "crates/diffy-semantic", @@ -136,6 +137,12 @@ codegen-units = 1 panic = "abort" strip = true +# The Node binding (bindings/node): release, but a Rust panic unwinds to the +# binding boundary and rejects the call instead of aborting the host process. +[profile.node] +inherits = "release" +panic = "unwind" + [patch.crates-io] accesskit_unix = { git = "https://github.com/seatedro/quark", rev = "b0d4abf67fb59d9e9fe66fbda78bda62bad160b5" } cosmic-text = { git = "https://github.com/seatedro/quark", rev = "b0d4abf67fb59d9e9fe66fbda78bda62bad160b5" } diff --git a/bindings/node/.gitignore b/bindings/node/.gitignore new file mode 100644 index 00000000..f85c308c --- /dev/null +++ b/bindings/node/.gitignore @@ -0,0 +1,5 @@ +*.node +node_modules/ +# Copied into each platform package at publish time. +npm/*/LICENSE-* +npm/*/THIRD_PARTY_NOTICES.md diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml new file mode 100644 index 00000000..9f30cd9d --- /dev/null +++ b/bindings/node/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "diffy-node" +description = "Node binding for diffy-compare, published as @diffy/node." +version.workspace = true +edition = "2024" +rust-version = "1.92" +license = "MIT OR Apache-2.0" +publish = false + +[lib] +crate-type = ["cdylib"] + +# Build with `--profile node` (see the workspace manifest): release panics +# abort, which would take the host Node process down with them. +[dependencies] +diffy-compare = { path = "../../crates/diffy-compare", features = ["serde"] } +# Grammars compiled in from $DIFFY_SYNTAX_BUNDLE; no packs, no network. +diffy-syntax = { path = "../../crates/diffy-syntax", features = ["bundled"] } +# Node-API bindings. `napi4` is the lowest version with thread-safe +# functions, which settle a promise from the request's own thread; +# `dyn-symbols` resolves Node-API from the host process at load time, so +# the same file loads in Node and Electron and needs no node.lib on Windows. +napi = { version = "3", default-features = false, features = ["napi4", "dyn-symbols"] } +# The `#[napi]` attribute. Without `type-def`: index.d.ts is written by +# hand from FORMAT.md, which generated types cannot express. +napi-derive = { version = "3", default-features = false, features = ["strict"] } +# Results cross into JS as one JSON string; already built by diffy-syntax. +serde_json = "1.0" + +[build-dependencies] +# Emits the platform link flags a Node addon needs (macOS dynamic lookup). +napi-build = "2" diff --git a/bindings/node/LICENSE-APACHE b/bindings/node/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/bindings/node/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bindings/node/LICENSE-MIT b/bindings/node/LICENSE-MIT new file mode 100644 index 00000000..2030f634 --- /dev/null +++ b/bindings/node/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Rohit (seatedro) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/node/README.md b/bindings/node/README.md new file mode 100644 index 00000000..a0057d5c --- /dev/null +++ b/bindings/node/README.md @@ -0,0 +1,108 @@ +# @diffy/node + +Diffy's file diff for Node. You read the files (from Git or anywhere else) +and pass in their contents. You get back a JSON document to render: a +semantic diff where a grammar exists, a line diff otherwise, with optional +syntax spans. The output format is specified in +[FORMAT.md](https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md), +and `index.d.ts` types every field. + +```js +const { diff, expand, languages, FORMAT_VERSION } = require('@diffy/node') + +const doc = await diff( + { + files: [{ status: 'modified', old_path: 'src/a.rs', new_path: 'src/a.rs', old: oldBytes, new: newBytes }], + syntax: true, + budget_ms: 2000, + }, + { signal: request.signal }, +) + +// Later, when the user expands a collapsed gap: +const rows = await expand(sameFileInput, doc.files[0].gap_after, { syntax: true }) +``` + +- `diff(request, { signal })` resolves to a `Document`, with files in the + order given. Contents are `Uint8Array`s (including `Buffer`s) or strings, + which are taken as UTF-8. + - `budget_ms` caps the whole request, counted from the call. Files that + start after it get line diffs, with `fallback.kind === "request_budget"`. + - `file_budget_ms` (default 400) caps each file's semantic diff. + - `syntax` adds syntax spans. `syntax_max_bytes` (default 1 MiB) skips + them for larger sides. +- `expand(file, gap, { syntax })` returns the context rows of a `Gap`, or of + any sub-range of one. `file` must have the contents the document was + made from. +- `languages()` lists the grammars compiled into this build. Every other + language gets a line diff. +- `FORMAT_VERSION` is the format version of the returned documents. + +Errors: + +- Aborting `signal` rejects at once with an `AbortError` whose `cause` is + the signal's reason. If the signal is already aborted, the call rejects + without starting any work. +- Malformed input rejects with a `TypeError` whose `code` is `"InvalidArg"`. + +## How it runs + +- Each call copies its input on the JS thread. The work then runs on a new + thread of its own (the diff fans out to a few more), not on libuv's + threadpool, so a long diff never holds up the server's file system, DNS, + or zlib calls. +- The result crosses into JS as a single JSON string, which is parsed + there. On an 11 MB document this cost about 75 ms on the JS thread (29 ms + to create the string, 45 ms to parse it). Building the same objects + through Node-API cost about 120 ms. +- Grammars are compiled in. Nothing is downloaded and nothing is read from + disk. +- Renames of identifiers are detected but not vetted against the rest of + the repository, which the binding never sees. +- The binding uses only Node-API (version 4), so one binary loads in Node + 18 and later and in Electron. +- A Rust panic rejects the call instead of crashing the process. A crash + inside a grammar's C parser or scanner still takes the process down. + +## Packages + +`@diffy/node` has no dependencies of its own. It pulls in one prebuilt +binary as an optional dependency: + +| Package | Platform | +| --- | --- | +| `@diffy/node-darwin-arm64` | macOS 11+, Apple silicon | +| `@diffy/node-darwin-x64` | macOS 11+, Intel | +| `@diffy/node-linux-x64-gnu` | Linux x64, glibc 2.28+ | +| `@diffy/node-linux-arm64-gnu` | Linux arm64, glibc 2.28+ | +| `@diffy/node-win32-x64-msvc` | Windows x64 | + +There is no musl build. Installing with `--omit=optional` leaves the binary +out. Each platform package ships `THIRD_PARTY_NOTICES.md`, which lists the +grammars compiled in and their licenses. + +For Electron, unpack `*.node` from the asar archive (for example with +electron-builder's `asarUnpack`). + +## Development + +From the repository root: + +```sh +# Grammars to compile in; rust and typescript are what the tests need. +cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" rust typescript tsx +cd bindings/node +DIFFY_SYNTAX_BUNDLE="$PWD/../../target/syntax-bundle" npm run build # writes diffy..node +npm test +``` + +The build uses the workspace's `node` profile: `release`, but with +`panic = "unwind"` so that panics can be caught. Set +`DIFFY_NODE_BINARY=path/to/diffy.node` to load a specific binary. + +`.github/workflows/node.yml` builds and tests every platform. Linux builds +run inside manylinux_2_28, so their binaries need glibc 2.28 at most, and +CI checks this. Pushing a `node-v` tag publishes the platform +packages and then `@diffy/node`. Before tagging, set that version in +`package.json`, in every `npm/*/package.json`, and in the +`optionalDependencies`. diff --git a/bindings/node/binding.js b/bindings/node/binding.js new file mode 100644 index 00000000..1cc504fc --- /dev/null +++ b/bindings/node/binding.js @@ -0,0 +1,41 @@ +'use strict' + +// Loads the native addon: $DIFFY_NODE_BINARY (a path, from the working +// directory) if set, else a local build beside this file (`npm run build`), +// else this platform's package. + +const { existsSync, readFileSync } = require('node:fs') +const { join, resolve } = require('node:path') + +function platformName() { + const { platform, arch } = process + if (platform === 'linux') return `linux-${arch}-${isMusl() ? 'musl' : 'gnu'}` + if (platform === 'win32') return `win32-${arch}-msvc` + return `${platform}-${arch}` +} + +function isMusl() { + try { + return readFileSync('/usr/bin/ldd', 'utf8').includes('musl') + } catch { + return false + } +} + +function load() { + if (process.env.DIFFY_NODE_BINARY) return require(resolve(process.env.DIFFY_NODE_BINARY)) + const name = platformName() + const local = join(__dirname, `diffy.${name}.node`) + if (existsSync(local)) return require(local) + try { + return require(`@diffy/node-${name}`) + } catch (error) { + throw new Error( + `@diffy/node has no binary for ${name}: install @diffy/node-${name} ` + + '(npm skips optional dependencies with --omit=optional), or set DIFFY_NODE_BINARY', + { cause: error }, + ) + } +} + +module.exports = load() diff --git a/bindings/node/build.rs b/bindings/node/build.rs new file mode 100644 index 00000000..0f1b0100 --- /dev/null +++ b/bindings/node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts new file mode 100644 index 00000000..9a1aa2f4 --- /dev/null +++ b/bindings/node/index.d.ts @@ -0,0 +1,274 @@ +// Types for @diffy/node. The output types follow Diffy's display format, +// version 1; the spec of every field is crates/diffy-compare/FORMAT.md: +// https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md +// +// Clients must ignore fields they do not know: adding an optional field +// does not change FORMAT_VERSION. + +/** The display format version of every Document this build returns. */ +export const FORMAT_VERSION: number + +/** Canonical names of the grammars compiled into this build, e.g. `"rust"`. */ +export function languages(): string[] + +/** + * Diffs every file off the JS thread. Resolves with one File per input, + * in input order. + * + * Rejects with an `AbortError` (`name === "AbortError"`, `code === + * "ABORT_ERR"`, `cause` the signal's reason) when `options.signal` aborts, + * and with a `TypeError` whose `code` is `"InvalidArg"` for malformed input. + */ +export function diff(request: DiffRequest, options?: DiffOptions): Promise + +/** + * The context rows of `gap` (a Gap from a Document, or any sub-range of + * one) of `file`, which must hold the same contents the Document was made + * from. Fewer rows than `gap.len` come back when the range runs past the + * end of either side or the sides no longer match there. Rejects with a + * `TypeError` whose `code` is `"InvalidArg"` for malformed input. + */ +export function expand(file: FileInput, gap: Gap, options?: ExpandOptions): Promise + +// ---- Input ------------------------------------------------------------- + +export type Status = 'modified' | 'added' | 'deleted' | 'renamed' | 'copied' + +/** + * One changed file as the caller read it. Contents are bytes; a string is + * taken as its UTF-8 encoding. Missing contents are empty. At least one + * path is required: the language comes from `new_path ?? old_path`. + */ +export interface FileInput { + status: Status + /** Absent for an added file. */ + old_path?: string + /** Absent for a deleted file. */ + new_path?: string + old?: Uint8Array | string + new?: Uint8Array | string +} + +export interface DiffRequest { + files: FileInput[] + /** + * Time budget for the whole request, from the call. Files started after + * it get line diffs, with `fallback.kind === "request_budget"`. + */ + budget_ms?: number + /** Time budget for each file's semantic diff. Default 400. */ + file_budget_ms?: number + /** Add syntax spans to rows. Default false. */ + syntax?: boolean + /** Sides longer than this, in bytes, get no syntax spans. Default 1 MiB. */ + syntax_max_bytes?: number +} + +export interface DiffOptions { + signal?: AbortSignal +} + +export interface ExpandOptions { + /** Add syntax spans to rows. Default false. */ + syntax?: boolean +} + +// ---- Output: the display format, version 1 ------------------------------ +// +// Line numbers are 1-based; ranges are half-open; span offsets are UTF-16 +// code units within the row's `text`, so `text.slice(start, end)` is the +// span. Absent arrays mean `[]`, absent booleans `false`. + +export interface Document { + version: 1 + /** In the order the files were requested. */ + files: File[] +} + +export interface File { + status: Status + /** Absent for an added file. */ + old_path?: string + /** Absent for a deleted file. Show `new_path ?? old_path`. */ + new_path?: string + /** + * The language tag the path names: the extension (`rs`, `ts`) or, for a + * file without one, its lowercased name (`dockerfile`). Absent when the + * path names none, or when `fallback.kind` is `no_grammar`. + */ + language?: string + /** A binary file has no `hunks` and no `gap_after`. */ + binary?: boolean + additions: number + deletions: number + /** The old file's last line has no newline. */ + old_missing_newline?: boolean + /** The new file's last line has no newline. */ + new_missing_newline?: boolean + /** Present when the file got a line diff instead of a semantic one. */ + fallback?: Fallback + /** In file order. Render each hunk's `gap_before`, header, rows; then `gap_after`. */ + hunks?: Hunk[] + /** + * Unchanged lines after the last hunk. For a file with no hunks + * (unchanged, or renamed without edits) this is the whole file. + */ + gap_after?: Gap + /** Items the semantic diff found changed: new-side items, then old-side ones. */ + outline?: OutlineItem[] + /** Code the semantic diff found moved. */ + moves?: Move[] + /** Identifiers the semantic diff found renamed. */ + renames?: Rename[] +} + +export interface Fallback { + /** + * `no_grammar`: no language is known for the path. `grammar_unavailable`: + * the path names a language whose grammar could not load. `semantic`: the + * semantic diff gave up, see `reason`. `request_budget`: the request's + * deadline passed before this file was diffed. + */ + kind: 'no_grammar' | 'grammar_unavailable' | 'semantic' | 'request_budget' + /** Present when `kind` is `semantic`. */ + reason?: 'too_large' | 'parse_errors' | 'error_coverage' | 'unparsed' | 'budget' | 'timeout' + /** For people; wording may change. Branch on `kind` and `reason`. */ + message: string +} + +export interface Hunk { + /** A unified diff's `@@ -old_start,old_len +new_start,new_len @@`. */ + old_start: number + old_len: number + new_start: number + new_len: number + /** The item enclosing the hunk's first change, for its header line. */ + header?: string + /** Unchanged lines between the previous hunk (or the top of the file) and this one. */ + gap_before?: Gap + /** In display order; each side's line numbers ascend. */ + rows: Row[] +} + +/** Unchanged lines left out of the hunks; identical on both sides. */ +export interface Gap { + old_start: number + new_start: number + /** At least 1. */ + len: number +} + +export type Row = ContextRow | RemovedRow | AddedRow | ModifiedRow + +/** An unchanged line. */ +export interface ContextRow { + kind: 'context' + old_line: number + new_line: number + text: string + /** From the new file. */ + syntax?: SyntaxSpan[] +} + +export interface RemovedRow { + kind: 'removed' + old: Line +} + +export interface AddedRow { + kind: 'added' + new: Line +} + +/** A removed line beside the added line it became. */ +export interface ModifiedRow { + kind: 'modified' + old: Line + new: Line +} + +/** One side of a changed row. */ +export interface Line { + /** Line number on its side. */ + line: number + /** No terminator, no trailing `\r`; invalid UTF-8 shown as U+FFFD. */ + text: string + /** What changed within the line. Sorted, never overlapping. */ + changes?: ChangeSpan[] + /** From this side's file. */ + syntax?: SyntaxSpan[] +} + +export interface ChangeSpan { + /** UTF-16 offset within the line's `text`. */ + start: number + end: number + /** + * `novel`: syntax only on this side; `novel_word`: a changed word in an + * otherwise matched line; `unchanged_context`: unchanged syntax inside a + * changed line; `renamed`: an identifier matched as renamed. + */ + intensity: 'novel' | 'novel_word' | 'unchanged_context' | 'renamed' +} + +export type SyntaxClass = + | 'keyword' + | 'string' + | 'comment' + | 'number' + | 'type' + | 'function' + | 'operator' + | 'punctuation' + | 'variable' + | 'constant' + | 'builtin' + | 'attribute' + | 'tag' + | 'property' + | 'namespace' + | 'label' + | 'preprocessor' + +export interface SyntaxSpan { + /** UTF-16 offset within the row's `text`. Sorted, never overlapping. */ + start: number + end: number + /** May gain values without a version bump; treat unknown ones as plain text. */ + class: SyntaxClass | (string & {}) +} + +export interface OutlineItem { + /** Which file `line` refers to: `removed` items are old, the rest new. */ + side: 'old' | 'new' + /** The item's first line. */ + line: number + /** The grammar's node kind, e.g. `function_item`. */ + kind: string + name: string + /** `renamed`: only identifiers inside it were renamed. */ + change: 'added' | 'removed' | 'renamed' | 'moved' | 'edited' + /** Changed lines of an `edited` item. */ + edits?: number +} + +/** `old_len` lines from old line `old_start` moved to `new_len` lines from new line `new_start`. */ +export interface Move { + old_start: number + old_len: number + new_start: number + new_len: number + /** Name of the item the code left. */ + from?: string + /** Name of the item the code arrived in. */ + into?: string + /** Name of the moved item itself, when the code is one. */ + item?: string +} + +export interface Rename { + old: string + new: string + /** Occurrences renamed. */ + uses: number +} diff --git a/bindings/node/index.js b/bindings/node/index.js new file mode 100644 index 00000000..78fab422 --- /dev/null +++ b/bindings/node/index.js @@ -0,0 +1,72 @@ +'use strict' + +const native = require('./binding.js') + +const FORMAT_VERSION = native.FORMAT_VERSION + +function languages() { + return native.languages() +} + +// Async, so argument errors reject like every other failure. +async function diff(request, options) { + const signal = options?.signal + if (typeof request !== 'object' || request === null) throw invalid('request must be an object') + if (signal !== undefined && typeof signal?.addEventListener !== 'function') { + throw invalid('options.signal must be an AbortSignal') + } + if (signal?.aborted) throw abortError(signal) + const token = new native.CancelToken() + const work = start(() => native.diff(request, token)) + if (signal === undefined) return JSON.parse(await work) + + // Abort rejects at once; the native request stops at its next file and + // its own rejection is dropped. + let onAbort + const aborted = new Promise((_, reject) => { + onAbort = () => { + token.cancel() + reject(abortError(signal)) + } + signal.addEventListener('abort', onAbort, { once: true }) + }) + work.catch(() => {}) + try { + return JSON.parse(await Promise.race([work, aborted])) + } catch (error) { + if (error?.code === 'Cancelled') throw abortError(signal) + throw error + } finally { + signal.removeEventListener('abort', onAbort) + } +} + +async function expand(file, gap, options) { + return JSON.parse(await start(() => native.expand(file, gap, options?.syntax))) +} + +// The binding throws synchronously only for its arguments (each conversion +// with its own code), or with GenericFailure when it cannot start a thread. +function start(call) { + try { + return call() + } catch (error) { + if (error?.code === 'GenericFailure') throw error + throw invalid(error?.message ?? String(error), error) + } +} + +function invalid(message, cause) { + const error = new TypeError(message, { cause }) + error.code = 'InvalidArg' + return error +} + +function abortError(signal) { + const error = new Error('The operation was aborted', { cause: signal.reason }) + error.name = 'AbortError' + error.code = 'ABORT_ERR' + return error +} + +module.exports = { FORMAT_VERSION, languages, diff, expand } diff --git a/bindings/node/npm/darwin-arm64/README.md b/bindings/node/npm/darwin-arm64/README.md new file mode 100644 index 00000000..d1b2eb61 --- /dev/null +++ b/bindings/node/npm/darwin-arm64/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-darwin-arm64` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Apple silicon. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. diff --git a/bindings/node/npm/darwin-arm64/package.json b/bindings/node/npm/darwin-arm64/package.json new file mode 100644 index 00000000..b8448e74 --- /dev/null +++ b/bindings/node/npm/darwin-arm64/package.json @@ -0,0 +1,31 @@ +{ + "name": "@diffy/node-darwin-arm64", + "version": "0.1.0", + "description": "The @diffy/node binary for macOS on Apple silicon.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/darwin-arm64" + }, + "main": "diffy.darwin-arm64.node", + "files": [ + "diffy.darwin-arm64.node", + "THIRD_PARTY_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/darwin-x64/README.md b/bindings/node/npm/darwin-x64/README.md new file mode 100644 index 00000000..38c45802 --- /dev/null +++ b/bindings/node/npm/darwin-x64/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-darwin-x64` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Intel. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. diff --git a/bindings/node/npm/darwin-x64/package.json b/bindings/node/npm/darwin-x64/package.json new file mode 100644 index 00000000..aebca37e --- /dev/null +++ b/bindings/node/npm/darwin-x64/package.json @@ -0,0 +1,31 @@ +{ + "name": "@diffy/node-darwin-x64", + "version": "0.1.0", + "description": "The @diffy/node binary for macOS on Intel.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/darwin-x64" + }, + "main": "diffy.darwin-x64.node", + "files": [ + "diffy.darwin-x64.node", + "THIRD_PARTY_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/linux-arm64-gnu/README.md b/bindings/node/npm/linux-arm64-gnu/README.md new file mode 100644 index 00000000..3c99474d --- /dev/null +++ b/bindings/node/npm/linux-arm64-gnu/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-linux-arm64-gnu` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux arm64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. diff --git a/bindings/node/npm/linux-arm64-gnu/package.json b/bindings/node/npm/linux-arm64-gnu/package.json new file mode 100644 index 00000000..0862c3aa --- /dev/null +++ b/bindings/node/npm/linux-arm64-gnu/package.json @@ -0,0 +1,34 @@ +{ + "name": "@diffy/node-linux-arm64-gnu", + "version": "0.1.0", + "description": "The @diffy/node binary for Linux arm64 with glibc 2.28 or newer.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/linux-arm64-gnu" + }, + "main": "diffy.linux-arm64-gnu.node", + "files": [ + "diffy.linux-arm64-gnu.node", + "THIRD_PARTY_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/linux-x64-gnu/README.md b/bindings/node/npm/linux-x64-gnu/README.md new file mode 100644 index 00000000..35392e9e --- /dev/null +++ b/bindings/node/npm/linux-x64-gnu/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-linux-x64-gnu` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux x64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. diff --git a/bindings/node/npm/linux-x64-gnu/package.json b/bindings/node/npm/linux-x64-gnu/package.json new file mode 100644 index 00000000..1b71a702 --- /dev/null +++ b/bindings/node/npm/linux-x64-gnu/package.json @@ -0,0 +1,34 @@ +{ + "name": "@diffy/node-linux-x64-gnu", + "version": "0.1.0", + "description": "The @diffy/node binary for Linux x64 with glibc 2.28 or newer.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/linux-x64-gnu" + }, + "main": "diffy.linux-x64-gnu.node", + "files": [ + "diffy.linux-x64-gnu.node", + "THIRD_PARTY_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/npm/win32-x64-msvc/README.md b/bindings/node/npm/win32-x64-msvc/README.md new file mode 100644 index 00000000..392bcd74 --- /dev/null +++ b/bindings/node/npm/win32-x64-msvc/README.md @@ -0,0 +1,5 @@ +# `@diffy/node-win32-x64-msvc` + +The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Windows x64. Install `@diffy/node`, not this package. + +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. diff --git a/bindings/node/npm/win32-x64-msvc/package.json b/bindings/node/npm/win32-x64-msvc/package.json new file mode 100644 index 00000000..a2c4b36e --- /dev/null +++ b/bindings/node/npm/win32-x64-msvc/package.json @@ -0,0 +1,31 @@ +{ + "name": "@diffy/node-win32-x64-msvc", + "version": "0.1.0", + "description": "The @diffy/node binary for Windows x64.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node/npm/win32-x64-msvc" + }, + "main": "diffy.win32-x64-msvc.node", + "files": [ + "diffy.win32-x64-msvc.node", + "THIRD_PARTY_NOTICES.md", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/package.json b/bindings/node/package.json new file mode 100644 index 00000000..c8b94837 --- /dev/null +++ b/bindings/node/package.json @@ -0,0 +1,47 @@ +{ + "name": "@diffy/node", + "version": "0.1.0", + "description": "Diffy's semantic file diff for Node: hand it file contents, get back a JSON diff to render.", + "license": "MIT OR Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/seatedro/diffy.git", + "directory": "bindings/node" + }, + "homepage": "https://github.com/seatedro/diffy/tree/master/bindings/node", + "keywords": ["diff", "semantic-diff", "tree-sitter", "git"], + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "index.js", + "index.d.ts", + "binding.js", + "LICENSE-MIT", + "LICENSE-APACHE" + ], + "engines": { + "node": ">=18" + }, + "scripts": { + "build": "node scripts/build.mjs", + "test": "node --test test/diffy.test.mjs" + }, + "optionalDependencies": { + "@diffy/node-darwin-arm64": "0.1.0", + "@diffy/node-darwin-x64": "0.1.0", + "@diffy/node-linux-x64-gnu": "0.1.0", + "@diffy/node-linux-arm64-gnu": "0.1.0", + "@diffy/node-win32-x64-msvc": "0.1.0" + }, + "publishConfig": { + "access": "public", + "provenance": true + } +} diff --git a/bindings/node/scripts/build.mjs b/bindings/node/scripts/build.mjs new file mode 100644 index 00000000..2c0b964f --- /dev/null +++ b/bindings/node/scripts/build.mjs @@ -0,0 +1,75 @@ +// Builds the addon with the `node` profile and copies it to +// `diffy..node`, the name binding.js and the platform packages use. +// +// node scripts/build.mjs [--target TRIPLE] [--out DIR] [--profile NAME] [--locked] +// +// Set DIFFY_SYNTAX_BUNDLE to the absolute `syntax-pack bundle --out` directory +// to compile grammars in; without it the build has none. + +import { spawnSync } from 'node:child_process' +import { copyFileSync, mkdirSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' + +const PLATFORMS = { + 'aarch64-apple-darwin': 'darwin-arm64', + 'x86_64-apple-darwin': 'darwin-x64', + 'x86_64-unknown-linux-gnu': 'linux-x64-gnu', + 'aarch64-unknown-linux-gnu': 'linux-arm64-gnu', + 'x86_64-pc-windows-msvc': 'win32-x64-msvc', +} + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const { values } = parseArgs({ + options: { + target: { type: 'string' }, + out: { type: 'string', default: root }, + profile: { type: 'string', default: 'node' }, + locked: { type: 'boolean', default: false }, + }, +}) + +const target = values.target ?? hostTriple() +const platform = PLATFORMS[target] +if (!platform) fail(`no npm platform for ${target}; known: ${Object.keys(PLATFORMS).join(', ')}`) +if (!process.env.DIFFY_SYNTAX_BUNDLE) console.warn('DIFFY_SYNTAX_BUNDLE is not set: no grammars') + +const args = ['build', '-p', 'diffy-node', '--profile', values.profile, '--message-format=json-render-diagnostics'] +if (values.target) args.push('--target', values.target) +if (values.locked) args.push('--locked') +const cargo = spawnSync('cargo', args, { + cwd: root, + stdio: ['ignore', 'pipe', 'inherit'], + encoding: 'utf8', + maxBuffer: 1 << 30, +}) +if (cargo.error) fail(`cargo: ${cargo.error.message}`) +if (cargo.status !== 0) fail(`cargo build failed (${cargo.status})`) + +let library +for (const line of cargo.stdout.split('\n')) { + if (!line.startsWith('{')) continue + const message = JSON.parse(line) + if (message.reason === 'compiler-artifact' && message.target?.name === 'diffy_node') { + library = message.filenames.find((file) => /\.(so|dylib|dll)$/.test(file)) ?? library + } +} +if (!library) fail('cargo reported no diffy_node library') + +mkdirSync(values.out, { recursive: true }) +const output = join(values.out, `diffy.${platform}.node`) +copyFileSync(library, output) +console.log(output) + +function hostTriple() { + const rustc = spawnSync('rustc', ['-vV'], { encoding: 'utf8' }) + const host = rustc.stdout?.match(/^host: (\S+)$/m)?.[1] + if (!host) fail('could not read the host triple from `rustc -vV`') + return host +} + +function fail(message) { + console.error(`build: ${message}`) + process.exit(1) +} diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs new file mode 100644 index 00000000..253dc939 --- /dev/null +++ b/bindings/node/src/lib.rs @@ -0,0 +1,273 @@ +//! The native half of `@diffy/node`; `index.js` wraps it and is the API. +//! +//! Each call copies its input on the JS thread, then does the work on a +//! thread of its own, never on libuv's pool (which a server's file system, +//! DNS, and zlib calls share), and settles its promise through a +//! thread-safe function. Results cross as one JSON string of the display +//! format, which `index.js` parses. + +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use diffy_compare::wire::{self, WireOptions}; +use diffy_compare::{FileStatus, GrammarStore, Options}; +use diffy_syntax::StoreConfig; +use napi::bindgen_prelude::{Either, Object, Uint8Array}; +use napi::{Env, Error, Result, Status}; +use napi_derive::napi; + +/// The semantic diff may run on the request thread itself; give it the +/// stack the app gives its semantic workers. Only reserved, not committed. +const STACK_BYTES: usize = 16 << 20; + +#[napi] +pub const FORMAT_VERSION: u32 = wire::FORMAT_VERSION; + +/// The grammars compiled into this build, by canonical name. +#[napi] +pub fn languages() -> Vec { + diffy_syntax::bundled::languages() + .map(str::to_owned) + .collect() +} + +/// Built on first use, off the JS thread; bundled grammars only, so it +/// never touches the network or the file system. +fn grammars() -> &'static GrammarStore { + static STORE: OnceLock = OnceLock::new(); + STORE.get_or_init(|| GrammarStore::new(StoreConfig::new().bundled())) +} + +#[napi(object, object_to_js = false)] +pub struct FileInput { + pub status: String, + #[napi(js_name = "old_path")] + pub old_path: Option, + #[napi(js_name = "new_path")] + pub new_path: Option, + pub old: Option>, + pub new: Option>, +} + +#[napi(object, object_to_js = false)] +pub struct DiffRequest { + pub files: Vec, + #[napi(js_name = "budget_ms")] + pub budget_ms: Option, + #[napi(js_name = "file_budget_ms")] + pub file_budget_ms: Option, + pub syntax: Option, + #[napi(js_name = "syntax_max_bytes")] + pub syntax_max_bytes: Option, +} + +#[napi(object, object_to_js = false)] +pub struct Gap { + #[napi(js_name = "old_start")] + pub old_start: f64, + #[napi(js_name = "new_start")] + pub new_start: f64, + pub len: f64, +} + +/// Stops the request it is passed to; `index.js` sets it from an +/// `AbortSignal`. +#[napi] +pub struct CancelToken { + flag: Arc, +} + +#[napi] +impl CancelToken { + #[napi(constructor)] + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + flag: Arc::new(AtomicBool::new(false)), + } + } + + #[napi] + pub fn cancel(&self) { + self.flag.store(true, Ordering::Relaxed); + } +} + +/// Diffs `request.files` and resolves to the display format's JSON. +#[napi] +pub fn diff<'env>( + env: &'env Env, + request: DiffRequest, + cancel: &CancelToken, +) -> Result> { + // The request budget runs from the call, not from when the thread starts. + let start = Instant::now(); + let deadline = request + .budget_ms + .map(|ms| duration("budget_ms", ms).map(|budget| start + budget)) + .transpose()?; + let file_budget = match request.file_budget_ms { + Some(ms) => duration("file_budget_ms", ms)?, + None => diffy_compare::FILE_BUDGET, + }; + let wire = wire_settings(request.syntax, request.syntax_max_bytes)?; + let inputs = request + .files + .into_iter() + .map(file_input) + .collect::>>()?; + let flag = Arc::clone(&cancel.flag); + spawn(env, "diffy-node-diff", move || { + let grammars = grammars(); + let mut options = Options::new(grammars); + options.file_budget = file_budget; + options.deadline = deadline; + options.cancel = Some(&flag); + // No `vet`: renames are detected but not confirmed against the rest + // of the repository, which the binding never sees. + let diffs = diffy_compare::diff_files(&inputs, &options).map_err(compare_error)?; + let document = wire::document(&inputs, &diffs, &wire.options(grammars)); + serde_json::to_string(&document).map_err(json_error) + }) +} + +/// The context rows of `gap` in `file`, as the display format's JSON. +#[napi] +pub fn expand<'env>( + env: &'env Env, + file: FileInput, + gap: Gap, + syntax: Option, +) -> Result> { + let gap = wire::Gap { + old_start: line_number("old_start", gap.old_start)?, + new_start: line_number("new_start", gap.new_start)?, + len: line_number("len", gap.len)?, + }; + let wire = wire_settings(syntax, None)?; + let input = file_input(file)?; + spawn(env, "diffy-node-expand", move || { + let rows = wire::expand(&input, &gap, &wire.options(grammars())); + serde_json::to_string(&rows).map_err(json_error) + }) +} + +/// [`WireOptions`] without the borrowed store, so it can cross threads. +#[derive(Clone, Copy)] +struct WireSettings { + syntax: bool, + syntax_max_bytes: usize, +} + +impl WireSettings { + fn options<'a>(&self, grammars: &'a GrammarStore) -> WireOptions<'a> { + WireOptions { + syntax: self.syntax.then_some(grammars), + syntax_max_bytes: self.syntax_max_bytes, + } + } +} + +fn wire_settings(syntax: Option, max_bytes: Option) -> Result { + let syntax_max_bytes = match max_bytes { + Some(bytes) if bytes.is_finite() && bytes >= 0.0 && bytes.fract() == 0.0 => { + // Saturates past usize::MAX, which means "no limit" anyway. + bytes as usize + } + Some(bytes) => return Err(invalid(format!("syntax_max_bytes: {bytes} is not a size"))), + None => wire::SYNTAX_MAX_BYTES, + }; + Ok(WireSettings { + syntax: syntax.unwrap_or(false), + syntax_max_bytes, + }) +} + +fn file_input(file: FileInput) -> Result { + let status = match file.status.as_str() { + "modified" => FileStatus::Modified, + "added" => FileStatus::Added, + "deleted" => FileStatus::Deleted, + "renamed" => FileStatus::Renamed, + "copied" => FileStatus::Copied, + other => return Err(invalid(format!("status: unknown file status {other:?}"))), + }; + if file.old_path.is_none() && file.new_path.is_none() { + return Err(invalid("a file needs old_path or new_path".to_owned())); + } + Ok(diffy_compare::FileInput { + status, + old_path: file.old_path, + new_path: file.new_path, + old: bytes(file.old), + new: bytes(file.new), + }) +} + +fn bytes(content: Option>) -> Vec { + match content { + Some(Either::A(array)) => array.to_vec(), + Some(Either::B(text)) => text.into_bytes(), + None => Vec::new(), + } +} + +fn duration(name: &str, ms: f64) -> Result { + Duration::try_from_secs_f64(ms / 1000.0) + .map_err(|_| invalid(format!("{name}: {ms} is not a duration in milliseconds"))) +} + +fn line_number(name: &str, value: f64) -> Result { + if value.is_finite() && value >= 0.0 && value.fract() == 0.0 && value <= f64::from(u32::MAX) { + Ok(value as u32) + } else { + Err(invalid(format!("{name}: {value} is not a line number"))) + } +} + +fn invalid(reason: String) -> Error { + Error::new(Status::InvalidArg, reason) +} + +/// `index.js` turns the `Cancelled` code into an `AbortError`. +fn compare_error(error: diffy_compare::Error) -> Error { + let status = match error { + diffy_compare::Error::Cancelled => Status::Cancelled, + _ => Status::GenericFailure, + }; + Error::new(status, error.to_string()) +} + +fn json_error(error: serde_json::Error) -> Error { + Error::new(Status::GenericFailure, error.to_string()) +} + +/// Runs `work` on a new thread and returns a promise of its result. A +/// panic in `work` rejects the promise; it cannot unwind into Node. +fn spawn<'env, F>(env: &'env Env, name: &str, work: F) -> Result> +where + F: FnOnce() -> Result + Send + 'static, +{ + let (deferred, promise) = env.create_deferred::()?; + std::thread::Builder::new() + .name(name.to_owned()) + .stack_size(STACK_BYTES) + .spawn(move || match catch_unwind(AssertUnwindSafe(work)) { + Ok(Ok(json)) => deferred.resolve(move |_| Ok(json)), + Ok(Err(error)) => deferred.reject(error), + Err(panic) => deferred.reject(panic_error(panic.as_ref())), + }) + .map_err(|error| Error::from_reason(format!("could not start a thread: {error}")))?; + Ok(promise) +} + +fn panic_error(panic: &(dyn std::any::Any + Send)) -> Error { + let message = panic + .downcast_ref::<&str>() + .copied() + .or_else(|| panic.downcast_ref::().map(String::as_str)) + .unwrap_or("unknown panic"); + Error::new(Status::GenericFailure, format!("diffy panicked: {message}")) +} diff --git a/bindings/node/test/diffy.test.mjs b/bindings/node/test/diffy.test.mjs new file mode 100644 index 00000000..a2680e92 --- /dev/null +++ b/bindings/node/test/diffy.test.mjs @@ -0,0 +1,251 @@ +// npm test (after `npm run build` with a bundle holding rust and typescript) + +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' + +import { createRequire } from 'node:module' + +import { FORMAT_VERSION, diff, expand, languages } from '../index.js' + +const native = createRequire(import.meta.url)('../binding.js') + +const OLD_RS = `use std::fmt; + +fn render(width: usize) -> String { + let label = "é😀"; + format!("{label} {width}") +} + +fn unchanged() {} +` +const NEW_RS = OLD_RS.replace('fn render(', 'fn render_preview(').replace('{width}', '{width}!') + +const modified = (path, old, next) => ({ status: 'modified', old_path: path, new_path: path, old, new: next }) + +/** Many files with real edits, so a request runs long enough to abort. */ +function bigRequest(count) { + const body = Array.from({ length: 400 }, (_, i) => `fn f${i}(x: u32) -> u32 { x + ${i} }`).join('\n') + const files = Array.from({ length: count }, (_, i) => + modified(`src/f${i}.rs`, body, body.replaceAll('x + 1', 'x * 2').replace('fn f3(', 'fn g3(')), + ) + return { files } +} + +describe('build', () => { + it('reports the format version', () => { + assert.equal(FORMAT_VERSION, 1) + }) + + it('lists the bundled grammars', () => { + const names = languages() + assert.ok(Array.isArray(names)) + for (const name of ['rust', 'typescript']) assert.ok(names.includes(name), `${name} is bundled`) + }) +}) + +describe('expand', () => { + const file = modified('src/view.rs', OLD_RS, OLD_RS) + + it('returns context rows', async () => { + const rows = await expand(file, { old_start: 3, new_start: 3, len: 2 }) + assert.deepEqual(rows, [ + { kind: 'context', old_line: 3, new_line: 3, text: 'fn render(width: usize) -> String {' }, + { kind: 'context', old_line: 4, new_line: 4, text: ' let label = "é😀";' }, + ]) + }) + + it('adds syntax spans in UTF-16 offsets', async () => { + const [fn, label] = await expand(file, { old_start: 3, new_start: 3, len: 2 }, { syntax: true }) + const at = (row, cls) => row.syntax.filter((s) => s.class === cls).map((s) => row.text.slice(s.start, s.end)) + assert.ok(at(fn, 'keyword').includes('fn')) + assert.ok(at(fn, 'function').includes('render')) + assert.ok(at(label, 'string').some((text) => text.includes('é😀'))) + }) + + it('takes bytes and strings alike', async () => { + const bytes = new TextEncoder().encode(OLD_RS) + // A view into a larger buffer: only the view's bytes count. + const padded = new Uint8Array(bytes.length + 8) + padded.set(bytes, 4) + const view = padded.subarray(4, 4 + bytes.length) + const gap = { old_start: 1, new_start: 1, len: 100 } + const want = await expand(file, gap) + assert.deepEqual(await expand({ ...file, old: bytes, new: Buffer.from(OLD_RS) }, gap), want) + assert.deepEqual(await expand({ ...file, old: view, new: view }, gap), want) + assert.equal(want.length, 8) + }) + + it('stops where the sides differ or end', async () => { + const changed = modified('a.rs', 'a\nb\nc\n', 'a\nB\nc\n') + assert.equal((await expand(changed, { old_start: 1, new_start: 1, len: 3 })).length, 1) + assert.equal((await expand(changed, { old_start: 3, new_start: 3, len: 10 })).length, 1) + assert.deepEqual(await expand(changed, { old_start: 1, new_start: 1, len: 0 }), []) + }) +}) + +describe('abort', () => { + it('rejects without starting when already aborted', async () => { + const controller = new AbortController() + const reason = new Error('navigated away') + controller.abort(reason) + await assert.rejects(diff(bigRequest(1), { signal: controller.signal }), (error) => { + assert.equal(error.name, 'AbortError') + assert.equal(error.code, 'ABORT_ERR') + assert.equal(error.cause, reason) + return true + }) + }) + + it('rejects a running request at once', async () => { + const controller = new AbortController() + const running = diff(bigRequest(64), { signal: controller.signal }) + controller.abort() + await assert.rejects(running, { name: 'AbortError' }) + }) + + // Needs the compare core: without it the request fails before the abort. + it('leaves the binding usable', async () => { + const controller = new AbortController() + const running = diff(bigRequest(256), { signal: controller.signal }) + setImmediate(() => controller.abort()) + await assert.rejects(running, { name: 'AbortError' }) + const rows = await expand(modified('a.rs', 'x\n', 'x\n'), { old_start: 1, new_start: 1, len: 1 }) + assert.equal(rows.length, 1) + }) +}) + +describe('invalid input rejects', () => { + const bad = { + 'no request': () => diff(), + 'null request': () => diff(null), + 'files not an array': () => diff({ files: 'a.rs' }), + 'unknown status': () => diff({ files: [{ status: 'changed', new_path: 'a.rs' }] }), + 'missing status': () => diff({ files: [{ new_path: 'a.rs' }] }), + 'no path': () => diff({ files: [{ status: 'modified', old: 'a', new: 'b' }] }), + 'contents a number': () => diff({ files: [{ status: 'added', new_path: 'a.rs', new: 42 }] }), + 'path a number': () => diff({ files: [{ status: 'added', new_path: 7 }] }), + 'negative budget': () => diff({ files: [], budget_ms: -1 }), + 'NaN file budget': () => diff({ files: [], file_budget_ms: NaN }), + 'fractional syntax_max_bytes': () => diff({ files: [], syntax_max_bytes: 1.5 }), + 'syntax not a boolean': () => diff({ files: [], syntax: 'yes' }), + 'signal not a signal': () => diff({ files: [] }, { signal: {} }), + 'negative gap start': () => expand(modified('a.rs', 'a', 'a'), { old_start: -1, new_start: 1, len: 1 }), + 'fractional gap length': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1, len: 1.5 }), + 'gap missing len': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1 }), + 'no gap': () => expand(modified('a.rs', 'a', 'a')), + } + for (const [name, call] of Object.entries(bad)) { + it(name, async () => { + const result = call() + assert.ok(result instanceof Promise, 'errors are rejections, not throws') + await assert.rejects(result, (error) => { + assert.ok(error instanceof Error) + assert.equal(error.code, 'InvalidArg', error.message) + return true + }) + }) + } +}) + +// These need the compare core (diffy_compare::diff_files). +describe('diff', () => { + it('diffs a modified file semantically', async () => { + const doc = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) + assert.equal(doc.version, FORMAT_VERSION) + assert.equal(doc.files.length, 1) + const [file] = doc.files + assert.equal(file.status, 'modified') + assert.equal(file.language, 'rs') + assert.equal(file.fallback, undefined) + assert.equal(file.additions, 2) + assert.equal(file.deletions, 2) + const rows = file.hunks.flatMap((hunk) => hunk.rows) + const changed = rows.filter((row) => row.kind === 'modified') + assert.equal(changed.length, 2) + const [renamed] = changed + assert.equal(renamed.old.text, 'fn render(width: usize) -> String {') + assert.equal(renamed.new.text, 'fn render_preview(width: usize) -> String {') + for (const span of renamed.new.changes ?? []) { + assert.ok(span.start < span.end && span.end <= renamed.new.text.length) + } + assert.ok(file.renames?.some((r) => r.old === 'render' && r.new === 'render_preview')) + }) + + it('adds syntax spans on request', async () => { + const plain = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) + const highlighted = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)], syntax: true }) + const spans = (doc) => + doc.files[0].hunks.flatMap((h) => h.rows).flatMap((row) => [row.syntax, row.old?.syntax, row.new?.syntax]) + assert.ok(spans(plain).every((s) => s === undefined)) + assert.ok(spans(highlighted).some((s) => s?.length > 0)) + }) + + it('keeps input order across statuses and kinds', async () => { + const files = [ + { status: 'added', new_path: 'notes.txt', new: 'hello' }, + { status: 'deleted', old_path: 'old.ts', old: 'export const a = 1\n' }, + modified('logo.png', new Uint8Array([0x89, 0x50, 0, 1]), new Uint8Array([0x89, 0x50, 0, 2])), + modified('data.unknownext', 'a\nb\n', 'a\nc\n'), + { status: 'renamed', old_path: 'a.ts', new_path: 'b.ts', old: 'let a = 1\n', new: 'let a = 1\n' }, + modified('bad.rs', Buffer.from([0x66, 0x6e, 0xff, 0x0a]), Buffer.from([0x66, 0x6f, 0xff, 0x0a])), + ] + const doc = await diff({ files }) + assert.deepEqual( + doc.files.map((f) => f.new_path ?? f.old_path), + ['notes.txt', 'old.ts', 'logo.png', 'data.unknownext', 'b.ts', 'bad.rs'], + ) + const [added, deleted, binary, unknown, renamed, invalid] = doc.files + assert.equal(added.status, 'added') + assert.equal(added.additions, 1) + assert.equal(added.new_missing_newline, true) + assert.equal(deleted.deletions, 1) + assert.equal(binary.binary, true) + assert.equal(binary.hunks, undefined) + // The extension may name no language or one with no grammar here. + assert.ok(['no_grammar', 'grammar_unavailable'].includes(unknown.fallback?.kind)) + assert.equal(renamed.status, 'renamed') + assert.equal(renamed.hunks, undefined) + assert.deepEqual(renamed.gap_after, { old_start: 1, new_start: 1, len: 1 }) + const texts = invalid.hunks.flatMap((h) => h.rows).flatMap((row) => [row.old?.text, row.new?.text]) + assert.ok(texts.includes('fn�')) + }) + + it('falls back to line diffs past the request budget', async () => { + const { files } = bigRequest(8) + const doc = await diff({ files, budget_ms: 0 }) + assert.equal(doc.files.length, 8) + for (const file of doc.files) { + assert.equal(file.fallback?.kind, 'request_budget') + assert.ok(file.hunks.length > 0) + } + }) + + it('expands the gaps it returns', async () => { + const doc = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) + const [file] = doc.files + const gaps = [...file.hunks.map((h) => h.gap_before), file.gap_after].filter(Boolean) + assert.ok(gaps.length > 0) + for (const gap of gaps) { + const rows = await expand(modified('src/view.rs', OLD_RS, NEW_RS), gap) + assert.equal(rows.length, gap.len) + } + }) + + it('runs requests concurrently', async () => { + const docs = await Promise.all(Array.from({ length: 8 }, () => diff(bigRequest(4)))) + for (const doc of docs) assert.equal(doc.files.length, 4) + }) + + it('stops the native request on cancel', async () => { + const token = new native.CancelToken() + token.cancel() + await assert.rejects(native.diff(bigRequest(256), token), { code: 'Cancelled' }) + }) + + it('finishes when the signal never aborts', async () => { + const controller = new AbortController() + const doc = await diff({ files: [modified('a.rs', 'a\n', 'b\n')] }, { signal: controller.signal }) + assert.equal(doc.files.length, 1) + controller.abort() + }) +}) From 43c4242243cc80e7f98d6800456a3fae7815606c Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:45:22 +0000 Subject: [PATCH 07/16] perf(compare): project files on workers and keep syntax inside the request budget wire::document highlighted every file serially after the diff, so an 876-file request with syntax took 10.9s and ran past budget_ms. Files are now projected on workers, and WireOptions gains a deadline and cancel flag past which files skip syntax spans and scope headers. The Node binding passes both; the gap test uses a file long enough to have gaps. --- bindings/node/README.md | 2 ++ bindings/node/index.d.ts | 2 ++ bindings/node/src/lib.rs | 11 ++++++- bindings/node/test/diffy.test.mjs | 12 ++++---- crates/diffy-compare/src/run.rs | 40 ++++++++++++++++++++++++++ crates/diffy-compare/src/wire.rs | 35 ++++++++++++++++++---- crates/diffy-compare/src/wire/tests.rs | 23 +++++++++++++++ 7 files changed, 114 insertions(+), 11 deletions(-) diff --git a/bindings/node/README.md b/bindings/node/README.md index a0057d5c..a9ee7026 100644 --- a/bindings/node/README.md +++ b/bindings/node/README.md @@ -28,6 +28,8 @@ const rows = await expand(sameFileInput, doc.files[0].gap_after, { syntax: true which are taken as UTF-8. - `budget_ms` caps the whole request, counted from the call. Files that start after it get line diffs, with `fallback.kind === "request_budget"`. + With `syntax`, files projected after it also come back without syntax + spans or scope headers. - `file_budget_ms` (default 400) caps each file's semantic diff. - `syntax` adds syntax spans. `syntax_max_bytes` (default 1 MiB) skips them for larger sides. diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 9a1aa2f4..41fb0e17 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -54,6 +54,8 @@ export interface DiffRequest { /** * Time budget for the whole request, from the call. Files started after * it get line diffs, with `fallback.kind === "request_budget"`. + * With `syntax`, files projected after it get no syntax spans or scope + * headers. */ budget_ms?: number /** Time budget for each file's semantic diff. Default 400. */ diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 253dc939..23f0f894 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -128,7 +128,15 @@ pub fn diff<'env>( // No `vet`: renames are detected but not confirmed against the rest // of the repository, which the binding never sees. let diffs = diffy_compare::diff_files(&inputs, &options).map_err(compare_error)?; - let document = wire::document(&inputs, &diffs, &wire.options(grammars)); + let document = wire::document( + &inputs, + &diffs, + &WireOptions { + deadline, + cancel: Some(&flag), + ..wire.options(grammars) + }, + ); serde_json::to_string(&document).map_err(json_error) }) } @@ -166,6 +174,7 @@ impl WireSettings { WireOptions { syntax: self.syntax.then_some(grammars), syntax_max_bytes: self.syntax_max_bytes, + ..WireOptions::default() } } } diff --git a/bindings/node/test/diffy.test.mjs b/bindings/node/test/diffy.test.mjs index a2680e92..79283e1f 100644 --- a/bindings/node/test/diffy.test.mjs +++ b/bindings/node/test/diffy.test.mjs @@ -221,12 +221,14 @@ describe('diff', () => { }) it('expands the gaps it returns', async () => { - const doc = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) - const [file] = doc.files - const gaps = [...file.hunks.map((h) => h.gap_before), file.gap_after].filter(Boolean) - assert.ok(gaps.length > 0) + // Unchanged functions on both ends, so context collapses before and after. + const pad = Array.from({ length: 12 }, (_, i) => `fn pad${i}() {}`).join('\n') + const file = modified('src/view.rs', `${pad}\n${OLD_RS}${pad}\n`, `${pad}\n${NEW_RS}${pad}\n`) + const doc = await diff({ files: [file] }) + const gaps = [...doc.files[0].hunks.map((h) => h.gap_before), doc.files[0].gap_after].filter(Boolean) + assert.equal(gaps.length, 2) for (const gap of gaps) { - const rows = await expand(modified('src/view.rs', OLD_RS, NEW_RS), gap) + const rows = await expand(file, gap) assert.equal(rows.length, gap.len) } }) diff --git a/crates/diffy-compare/src/run.rs b/crates/diffy-compare/src/run.rs index 3c8a917f..b3878675 100644 --- a/crates/diffy-compare/src/run.rs +++ b/crates/diffy-compare/src/run.rs @@ -101,6 +101,46 @@ pub(crate) fn diff_all( done.into_iter().map(|(_, diff)| diff).collect() } +/// Runs `work` for every index in `0..len` on worker threads and returns +/// the results in index order. The calling thread works too, so a worker +/// that cannot start only costs parallelism; a worker's panic is resumed +/// on the caller. +#[cfg(feature = "serde")] +pub(crate) fn map_parallel(len: usize, work: &(dyn Fn(usize) -> T + Sync)) -> Vec { + let next = AtomicUsize::new(0); + let run = || { + let mut done = Vec::new(); + loop { + let at = next.fetch_add(1, Ordering::Relaxed); + if at >= len { + return done; + } + done.push((at, work(at))); + } + }; + let mut done = std::thread::scope(|scope| { + let handles: Vec<_> = (1..worker_count(len)) + .filter_map(|index| { + std::thread::Builder::new() + .name(format!("diffy-wire-{index}")) + .stack_size(WORKER_STACK_BYTES) + .spawn_scoped(scope, run) + .ok() + }) + .collect(); + let mut done = run(); + for handle in handles { + match handle.join() { + Ok(part) => done.extend(part), + Err(panic) => std::panic::resume_unwind(panic), + } + } + done + }); + done.sort_unstable_by_key(|(at, _)| *at); + done.into_iter().map(|(_, value)| value).collect() +} + fn worker_count(files: usize) -> usize { if files < MIN_FILES_FOR_PARALLEL { return 1; diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs index 31a4cdf7..bc50731e 100644 --- a/crates/diffy-compare/src/wire.rs +++ b/crates/diffy-compare/src/wire.rs @@ -24,6 +24,8 @@ use std::borrow::Cow; use std::cell::OnceCell; use std::ops::Range; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; use diffy_diff::{ AlignedRow, BlockKind, ChangeIntensity, DiffDocument, FileStatus, ItemChange, Side as DocSide, @@ -49,6 +51,11 @@ pub struct WireOptions<'a> { /// Sides longer than this, in bytes, get no syntax spans or scope /// headers. pub syntax_max_bytes: usize, + /// Files projected after this get no syntax spans or scope headers, so + /// highlighting cannot run a request past its budget. + pub deadline: Option, + /// Once set, the remaining files get no syntax spans or scope headers. + pub cancel: Option<&'a AtomicBool>, } impl Default for WireOptions<'_> { @@ -56,6 +63,8 @@ impl Default for WireOptions<'_> { Self { syntax: None, syntax_max_bytes: SYNTAX_MAX_BYTES, + deadline: None, + cancel: None, } } } @@ -379,19 +388,35 @@ fn is_false(value: &bool) -> bool { /// Projects every file; `inputs` and `diffs` pair up by index, and extra /// entries of the longer slice are ignored. pub fn document(inputs: &[FileInput], diffs: &[FileDiff], options: &WireOptions<'_>) -> Document { + let len = inputs.len().min(diffs.len()); + // Highlighting parses both sides of every file, so files are projected + // on workers like the diff itself. + let files = if options.syntax.is_some() { + crate::run::map_parallel(len, &|at| file(&inputs[at], &diffs[at], options)) + } else { + (0..len) + .map(|at| file(&inputs[at], &diffs[at], options)) + .collect() + }; Document { version: FORMAT_VERSION, - files: inputs - .iter() - .zip(diffs) - .map(|(input, diff)| file(input, diff, options)) - .collect(), + files, } } /// Projects one file. Highlighting and scope headers parse each side once; /// everything else is linear in the rows shown. pub fn file(input: &FileInput, diff: &FileDiff, options: &WireOptions<'_>) -> File { + let late = options + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + || options + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)); + let options = &WireOptions { + syntax: options.syntax.filter(|_| !late), + ..*options + }; let doc = &diff.document; let has_file = doc.file_count() > 0; let binary = crate::is_binary(&input.old) diff --git a/crates/diffy-compare/src/wire/tests.rs b/crates/diffy-compare/src/wire/tests.rs index 5e6b08aa..7a86ab15 100644 --- a/crates/diffy-compare/src/wire/tests.rs +++ b/crates/diffy-compare/src/wire/tests.rs @@ -691,6 +691,7 @@ fn syntax_spans_are_classes_sliced_per_line_in_utf16() { let capped = WireOptions { syntax: Some(&store), syntax_max_bytes: 8, + ..WireOptions::default() }; let file = super::file(&input, &diff, &capped); assert_eq!(file.hunks[0].header, None); @@ -747,3 +748,25 @@ fn aligned_rows_that_miss_lines_fall_back_to_positional_pairs() { assert_eq!((old_line.line, new_line.line), (number, number)); } } + +#[test] +fn files_projected_past_the_deadline_skip_syntax() { + let Some(store) = diffy_syntax::testing::store_with("rust") else { + return; + }; + let (input, diff) = rust_change(); + let options = WireOptions { + syntax: Some(&store), + deadline: Some(std::time::Instant::now()), + ..WireOptions::default() + }; + let doc = document(&[input.clone(), input], &[diff.clone(), diff], &options); + for file in &doc.files { + assert!(file.hunks[0].rows.iter().all(|row| match row { + Row::Context { syntax, .. } => syntax.is_empty(), + Row::Removed { old: line } | Row::Added { new: line } => line.syntax.is_empty(), + Row::Modified { old, new } => old.syntax.is_empty() && new.syntax.is_empty(), + })); + assert!(!file.hunks.is_empty()); + } +} From de1a667f2665c6576aba9831784a927d8f454b17 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 05:57:34 +0000 Subject: [PATCH 08/16] feat(node): queue diff requests and add diffJson diff calls now wait in a process-wide FIFO queue: at most max_concurrent_requests run at once (default: available cores / 4, at least 1, since a request fans out to four workers). Waiting holds no thread; a request's thread takes the next queued one when it finishes. A request aborted while queued rejects at once and never starts, and budget_ms keeps counting from the call. expand is not queued. configure({ max_concurrent_requests }) sets the limit. diffJson resolves to the document's JSON text, unparsed, for servers that forward it. --- bindings/node/README.md | 40 ++++-- bindings/node/index.d.ts | 31 ++++- bindings/node/index.js | 32 +++-- bindings/node/src/lib.rs | 99 ++++++++------ bindings/node/src/queue.rs | 213 ++++++++++++++++++++++++++++++ bindings/node/test/diffy.test.mjs | 132 +++++++++++++++++- 6 files changed, 480 insertions(+), 67 deletions(-) create mode 100644 bindings/node/src/queue.rs diff --git a/bindings/node/README.md b/bindings/node/README.md index a9ee7026..e481a594 100644 --- a/bindings/node/README.md +++ b/bindings/node/README.md @@ -8,7 +8,7 @@ syntax spans. The output format is specified in and `index.d.ts` types every field. ```js -const { diff, expand, languages, FORMAT_VERSION } = require('@diffy/node') +const { diff, diffJson, expand, configure, languages, FORMAT_VERSION } = require('@diffy/node') const doc = await diff( { @@ -26,16 +26,24 @@ const rows = await expand(sameFileInput, doc.files[0].gap_after, { syntax: true - `diff(request, { signal })` resolves to a `Document`, with files in the order given. Contents are `Uint8Array`s (including `Buffer`s) or strings, which are taken as UTF-8. - - `budget_ms` caps the whole request, counted from the call. Files that - start after it get line diffs, with `fallback.kind === "request_budget"`. + - `budget_ms` caps the whole request, counted from the call, so time + spent waiting in the queue (see below) counts too. Files that start + after it get line diffs, with `fallback.kind === "request_budget"`. With `syntax`, files projected after it also come back without syntax spans or scope headers. - `file_budget_ms` (default 400) caps each file's semantic diff. - `syntax` adds syntax spans. `syntax_max_bytes` (default 1 MiB) skips them for larger sides. +- `diffJson(request, { signal })` is `diff`, but resolves to the document's + JSON text instead of parsing it. A server that only forwards the document + saves a `JSON.parse` and a `JSON.stringify` on its event loop (parsing a + 16 MB document blocks it for about 160 ms). - `expand(file, gap, { syntax })` returns the context rows of a `Gap`, or of any sub-range of one. `file` must have the contents the document was made from. +- `configure({ max_concurrent_requests })` sets how many `diff` and + `diffJson` calls run at once, process-wide, and returns the settings; + `configure()` only returns them. See below. - `languages()` lists the grammars compiled into this build. Every other language gets a line diff. - `FORMAT_VERSION` is the format version of the returned documents. @@ -46,17 +54,27 @@ Errors: the signal's reason. If the signal is already aborted, the call rejects without starting any work. - Malformed input rejects with a `TypeError` whose `code` is `"InvalidArg"`. + `configure` throws the same error. ## How it runs -- Each call copies its input on the JS thread. The work then runs on a new - thread of its own (the diff fans out to a few more), not on libuv's - threadpool, so a long diff never holds up the server's file system, DNS, - or zlib calls. -- The result crosses into JS as a single JSON string, which is parsed - there. On an 11 MB document this cost about 75 ms on the JS thread (29 ms - to create the string, 45 ms to parse it). Building the same objects - through Node-API cost about 120 ms. +- Each call copies its input on the JS thread. The work then runs on a + thread of its own, not on libuv's threadpool, so a long diff never holds + up the server's file system, DNS, or zlib calls. +- A diff of four or more files fans out to four worker threads, so `diff` + calls share one FIFO queue: at most `max_concurrent_requests` run at + once, by default the available cores divided by 4 (at least 1), which + keeps every core busy without time-slicing each other's file budgets. + Waiting holds no thread. A call aborted while it waits rejects at once + and never starts. Diffs of one to three files use one thread each, so a + server that sends files one at a time may want a higher limit. +- `expand` is not queued: it is one file on one thread, and someone is + waiting on it, so it should not wait behind whole diffs. +- The result crosses into JS as a single JSON string, which `diff` and + `expand` parse there. On an 11 MB document this cost about 75 ms on the + JS thread (29 ms to create the string, 45 ms to parse it). Building the + same objects through Node-API cost about 120 ms. `diffJson` skips the + parse. - Grammars are compiled in. Nothing is downloaded and nothing is read from disk. - Renames of identifiers are detected but not vetted against the rest of diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 41fb0e17..78150c9b 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -13,7 +13,8 @@ export function languages(): string[] /** * Diffs every file off the JS thread. Resolves with one File per input, - * in input order. + * in input order. Waits in a process-wide FIFO queue while + * `max_concurrent_requests` others run (see `configure`). * * Rejects with an `AbortError` (`name === "AbortError"`, `code === * "ABORT_ERR"`, `cause` the signal's reason) when `options.signal` aborts, @@ -21,6 +22,13 @@ export function languages(): string[] */ export function diff(request: DiffRequest, options?: DiffOptions): Promise +/** + * `diff`, resolving to the Document as the JSON text it was built as, not + * parsed: for a server forwarding it to clients without parsing it on its + * event loop. `JSON.parse` of it equals what `diff` resolves to. + */ +export function diffJson(request: DiffRequest, options?: DiffOptions): Promise + /** * The context rows of `gap` (a Gap from a Document, or any sub-range of * one) of `file`, which must hold the same contents the Document was made @@ -30,6 +38,22 @@ export function diff(request: DiffRequest, options?: DiffOptions): Promise +/** + * Sets the given options, process-wide, and returns all of them. Throws a + * `TypeError` whose `code` is `"InvalidArg"` for an invalid value. + */ +export function configure(options?: Configuration): Required + +export interface Configuration { + /** + * The most `diff` and `diffJson` calls that run at once; later ones wait + * their turn. A positive integer. Default: the available cores divided + * by 4, at least 1. Raising it starts waiting requests at once; lowering + * it lets running ones finish. `expand` is never queued. + */ + max_concurrent_requests?: number +} + // ---- Input ------------------------------------------------------------- export type Status = 'modified' | 'added' | 'deleted' | 'renamed' | 'copied' @@ -52,8 +76,9 @@ export interface FileInput { export interface DiffRequest { files: FileInput[] /** - * Time budget for the whole request, from the call. Files started after - * it get line diffs, with `fallback.kind === "request_budget"`. + * Time budget for the whole request, from the call, so it includes time + * queued. Files started after it get line diffs, with + * `fallback.kind === "request_budget"`. * With `syntax`, files projected after it get no syntax spans or scope * headers. */ diff --git a/bindings/node/index.js b/bindings/node/index.js index 78fab422..4aaa5270 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -8,8 +8,12 @@ function languages() { return native.languages() } -// Async, so argument errors reject like every other failure. async function diff(request, options) { + return JSON.parse(await diffJson(request, options)) +} + +// Async, so argument errors reject like every other failure. +async function diffJson(request, options) { const signal = options?.signal if (typeof request !== 'object' || request === null) throw invalid('request must be an object') if (signal !== undefined && typeof signal?.addEventListener !== 'function') { @@ -17,11 +21,11 @@ async function diff(request, options) { } if (signal?.aborted) throw abortError(signal) const token = new native.CancelToken() - const work = start(() => native.diff(request, token)) - if (signal === undefined) return JSON.parse(await work) + const work = call(() => native.diff(request, token)) + if (signal === undefined) return work - // Abort rejects at once; the native request stops at its next file and - // its own rejection is dropped. + // Abort rejects at once; a queued native request never starts, a + // running one stops at its next file, and its own rejection is dropped. let onAbort const aborted = new Promise((_, reject) => { onAbort = () => { @@ -32,7 +36,7 @@ async function diff(request, options) { }) work.catch(() => {}) try { - return JSON.parse(await Promise.race([work, aborted])) + return await Promise.race([work, aborted]) } catch (error) { if (error?.code === 'Cancelled') throw abortError(signal) throw error @@ -42,14 +46,22 @@ async function diff(request, options) { } async function expand(file, gap, options) { - return JSON.parse(await start(() => native.expand(file, gap, options?.syntax))) + return JSON.parse(await call(() => native.expand(file, gap, options?.syntax))) +} + +function configure(options) { + if (options !== undefined && (typeof options !== 'object' || options === null)) { + throw invalid('options must be an object') + } + const state = call(() => native.queue(options?.max_concurrent_requests)) + return { max_concurrent_requests: state.limit } } // The binding throws synchronously only for its arguments (each conversion // with its own code), or with GenericFailure when it cannot start a thread. -function start(call) { +function call(binding) { try { - return call() + return binding() } catch (error) { if (error?.code === 'GenericFailure') throw error throw invalid(error?.message ?? String(error), error) @@ -69,4 +81,4 @@ function abortError(signal) { return error } -module.exports = { FORMAT_VERSION, languages, diff, expand } +module.exports = { FORMAT_VERSION, languages, configure, diff, diffJson, expand } diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 23f0f894..81f0e5f0 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -3,10 +3,10 @@ //! Each call copies its input on the JS thread, then does the work on a //! thread of its own, never on libuv's pool (which a server's file system, //! DNS, and zlib calls share), and settles its promise through a -//! thread-safe function. Results cross as one JSON string of the display -//! format, which `index.js` parses. +//! thread-safe function. `diff` requests wait their turn in [`queue`]; +//! `expand` starts at once. Results cross as one JSON string of the +//! display format. -use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -18,9 +18,7 @@ use napi::bindgen_prelude::{Either, Object, Uint8Array}; use napi::{Env, Error, Result, Status}; use napi_derive::napi; -/// The semantic diff may run on the request thread itself; give it the -/// stack the app gives its semantic workers. Only reserved, not committed. -const STACK_BYTES: usize = 16 << 20; +mod queue; #[napi] pub const FORMAT_VERSION: u32 = wire::FORMAT_VERSION; @@ -89,20 +87,54 @@ impl CancelToken { } } + /// Stops the request: a queued one is rejected at once and never + /// starts; a running one stops at its next file. #[napi] pub fn cancel(&self) { self.flag.store(true, Ordering::Relaxed); + queue::cancel(&self.flag); } } -/// Diffs `request.files` and resolves to the display format's JSON. +#[napi(object, object_from_js = false)] +pub struct QueueState { + /// The most `diff` requests that run at once. + pub limit: u32, + pub running: u32, + pub queued: u32, +} + +/// Sets the most `diff` requests that run at once, if `limit` is given, +/// and reports the queue. +#[napi(js_name = "queue")] +pub fn queue_state(limit: Option) -> Result { + if let Some(limit) = limit { + if !(limit.is_finite() && limit >= 1.0 && limit.fract() == 0.0) { + return Err(invalid(format!( + "max_concurrent_requests: {limit} is not a positive integer" + ))); + } + // Saturates past usize::MAX, which means "no limit" anyway. + queue::set_limit(limit as usize); + } + let state = queue::state(); + let count = |n: usize| u32::try_from(n).unwrap_or(u32::MAX); + Ok(QueueState { + limit: count(state.limit), + running: count(state.running), + queued: count(state.queued), + }) +} + +/// Diffs `request.files` and resolves to the display format's JSON, once +/// the request's turn in the queue comes. #[napi] pub fn diff<'env>( env: &'env Env, request: DiffRequest, cancel: &CancelToken, ) -> Result> { - // The request budget runs from the call, not from when the thread starts. + // The request budget runs from the call, so it counts time queued. let start = Instant::now(); let deadline = request .budget_ms @@ -119,7 +151,8 @@ pub fn diff<'env>( .map(file_input) .collect::>>()?; let flag = Arc::clone(&cancel.flag); - spawn(env, "diffy-node-diff", move || { + let (deferred, promise) = env.create_deferred()?; + let work: queue::Work = Box::new(move || { let grammars = grammars(); let mut options = Options::new(grammars); options.file_budget = file_budget; @@ -138,10 +171,17 @@ pub fn diff<'env>( }, ); serde_json::to_string(&document).map_err(json_error) - }) + }); + queue::submit(queue::Job { + deferred, + cancel: Arc::clone(&cancel.flag), + work, + }); + Ok(promise) } /// The context rows of `gap` in `file`, as the display format's JSON. +/// Never queued: it is one file on one thread, and a user waits on it. #[napi] pub fn expand<'env>( env: &'env Env, @@ -156,10 +196,17 @@ pub fn expand<'env>( }; let wire = wire_settings(syntax, None)?; let input = file_input(file)?; - spawn(env, "diffy-node-expand", move || { + let (deferred, promise) = env.create_deferred()?; + let work: queue::Work = Box::new(move || { let rows = wire::expand(&input, &gap, &wire.options(grammars())); serde_json::to_string(&rows).map_err(json_error) - }) + }); + std::thread::Builder::new() + .name("diffy-node-expand".to_owned()) + .stack_size(queue::STACK_BYTES) + .spawn(move || queue::settle(deferred, work)) + .map_err(|error| Error::from_reason(format!("could not start a thread: {error}")))?; + Ok(promise) } /// [`WireOptions`] without the borrowed store, so it can cross threads. @@ -252,31 +299,3 @@ fn compare_error(error: diffy_compare::Error) -> Error { fn json_error(error: serde_json::Error) -> Error { Error::new(Status::GenericFailure, error.to_string()) } - -/// Runs `work` on a new thread and returns a promise of its result. A -/// panic in `work` rejects the promise; it cannot unwind into Node. -fn spawn<'env, F>(env: &'env Env, name: &str, work: F) -> Result> -where - F: FnOnce() -> Result + Send + 'static, -{ - let (deferred, promise) = env.create_deferred::()?; - std::thread::Builder::new() - .name(name.to_owned()) - .stack_size(STACK_BYTES) - .spawn(move || match catch_unwind(AssertUnwindSafe(work)) { - Ok(Ok(json)) => deferred.resolve(move |_| Ok(json)), - Ok(Err(error)) => deferred.reject(error), - Err(panic) => deferred.reject(panic_error(panic.as_ref())), - }) - .map_err(|error| Error::from_reason(format!("could not start a thread: {error}")))?; - Ok(promise) -} - -fn panic_error(panic: &(dyn std::any::Any + Send)) -> Error { - let message = panic - .downcast_ref::<&str>() - .copied() - .or_else(|| panic.downcast_ref::().map(String::as_str)) - .unwrap_or("unknown panic"); - Error::new(Status::GenericFailure, format!("diffy panicked: {message}")) -} diff --git a/bindings/node/src/queue.rs b/bindings/node/src/queue.rs new file mode 100644 index 00000000..66b121c0 --- /dev/null +++ b/bindings/node/src/queue.rs @@ -0,0 +1,213 @@ +//! The process-wide FIFO queue of `diff` requests. +//! +//! At most [`limit`](Queue::limit) requests run at once, each on a thread +//! of its own that fans out to a few more. A queued request holds no thread +//! and blocks nothing: when a request finishes, its thread takes the next +//! queued one, or exits. Requests are cancelled out of the queue at once +//! and never start. + +use std::collections::VecDeque; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; + +use napi::{Env, Error, JsDeferred, Result, Status}; + +/// The semantic diff may run on the request thread itself; give it the +/// stack the app gives its semantic workers. Only reserved, not committed. +pub(crate) const STACK_BYTES: usize = 16 << 20; + +/// `diffy_compare` diffs a request of four or more files on four workers, +/// then highlights them on as many, so a request can keep this many cores +/// busy. +const WORKERS_PER_REQUEST: usize = 4; + +/// A promise settled with a JSON string from any thread. +pub(crate) type Deferred = JsDeferred Result + Send>>; + +/// A request's work: the JSON it resolves to. +pub(crate) type Work = Box Result + Send>; + +pub(crate) struct Job { + pub deferred: Deferred, + pub cancel: Arc, + pub work: Work, +} + +struct Queue { + /// 0 until first used. + limit: usize, + running: usize, + waiting: VecDeque, +} + +static QUEUE: Mutex = Mutex::new(Queue { + limit: 0, + running: 0, + waiting: VecDeque::new(), +}); + +fn lock() -> MutexGuard<'static, Queue> { + // Nothing panics while the lock is held, so the queue is consistent. + QUEUE.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl Queue { + fn limit(&mut self) -> usize { + if self.limit == 0 { + self.limit = default_limit(); + } + self.limit + } + + /// The oldest waiting job, counted as running, if a slot is free. + /// Cancelled jobs found on the way are rejected. + fn next(&mut self) -> Option { + if self.running >= self.limit() { + return None; + } + while let Some(job) = self.waiting.pop_front() { + if job.cancel.load(Ordering::Relaxed) { + reject_cancelled(job); + continue; + } + self.running += 1; + return Some(job); + } + None + } +} + +/// One request per [`WORKERS_PER_REQUEST`] cores, so running requests keep +/// every core busy without time-slicing each other's file budgets away. +fn default_limit() -> usize { + let cores = std::thread::available_parallelism().map_or(1, usize::from); + (cores / WORKERS_PER_REQUEST).max(1) +} + +/// Queues `job`; it starts once it is the oldest and fewer than the limit +/// are running. +pub(crate) fn submit(job: Job) { + let mut queue = lock(); + queue.waiting.push_back(job); + let ready = queue.next(); + drop(queue); + if let Some(job) = ready { + start(job); + } +} + +/// Sets the limit and starts the queued jobs it now lets run. Running jobs +/// past a lowered limit finish; no new ones start until under it. +pub(crate) fn set_limit(limit: usize) { + let mut queue = lock(); + queue.limit = limit.max(1); + let mut ready = Vec::new(); + while let Some(job) = queue.next() { + ready.push(job); + } + drop(queue); + for job in ready { + start(job); + } +} + +pub(crate) struct State { + pub limit: usize, + pub running: usize, + pub queued: usize, +} + +pub(crate) fn state() -> State { + let mut queue = lock(); + State { + limit: queue.limit(), + running: queue.running, + queued: queue.waiting.len(), + } +} + +/// Rejects the queued jobs `cancel` stops, which then never start. Running +/// ones see the flag themselves. +pub(crate) fn cancel(cancel: &Arc) { + let mut queue = lock(); + let waiting = std::mem::take(&mut queue.waiting); + for job in waiting { + if Arc::ptr_eq(&job.cancel, cancel) { + reject_cancelled(job); + } else { + queue.waiting.push_back(job); + } + } +} + +fn reject_cancelled(job: Job) { + job.deferred + .reject(Error::new(Status::Cancelled, "cancelled".to_owned())); +} + +/// Gives up the caller's slot, or hands it the next queued job. +fn finish() -> Option { + let mut queue = lock(); + queue.running = queue.running.saturating_sub(1); + queue.next() +} + +/// Runs `job`, which holds a slot, and then queued jobs, on a new thread. +fn start(mut job: Job) { + loop { + // The job is sent once the thread exists: a failed spawn drops its + // closure, and the job's promise must still be settled. + let error = match spawn_runner() { + Ok(sender) => match sender.send(job) { + Ok(()) => return, + Err(mpsc::SendError(unsent)) => { + job = unsent; + "it exited at once".to_owned() + } + }, + Err(error) => error.to_string(), + }; + job.deferred.reject(Error::from_reason(format!( + "could not start a thread: {error}" + ))); + match finish() { + Some(next) => job = next, + None => return, + } + } +} + +fn spawn_runner() -> std::io::Result> { + let (sender, receiver) = mpsc::sync_channel::(1); + std::thread::Builder::new() + .name("diffy-node-diff".to_owned()) + .stack_size(STACK_BYTES) + .spawn(move || { + let mut next = receiver.recv().ok(); + while let Some(job) = next { + settle(job.deferred, job.work); + next = finish(); + } + })?; + Ok(sender) +} + +/// Runs `work` and settles `deferred` with its result. A panic in `work` +/// rejects the promise; it cannot unwind into Node. +pub(crate) fn settle(deferred: Deferred, work: Work) { + match catch_unwind(AssertUnwindSafe(work)) { + Ok(Ok(json)) => deferred.resolve(Box::new(move |_| Ok(json))), + Ok(Err(error)) => deferred.reject(error), + Err(panic) => deferred.reject(panic_error(panic.as_ref())), + } +} + +fn panic_error(panic: &(dyn std::any::Any + Send)) -> Error { + let message = panic + .downcast_ref::<&str>() + .copied() + .or_else(|| panic.downcast_ref::().map(String::as_str)) + .unwrap_or("unknown panic"); + Error::new(Status::GenericFailure, format!("diffy panicked: {message}")) +} diff --git a/bindings/node/test/diffy.test.mjs b/bindings/node/test/diffy.test.mjs index 79283e1f..eb8dcb5b 100644 --- a/bindings/node/test/diffy.test.mjs +++ b/bindings/node/test/diffy.test.mjs @@ -1,11 +1,11 @@ // npm test (after `npm run build` with a bundle holding rust and typescript) import assert from 'node:assert/strict' -import { describe, it } from 'node:test' - import { createRequire } from 'node:module' +import { availableParallelism } from 'node:os' +import { afterEach, describe, it } from 'node:test' -import { FORMAT_VERSION, diff, expand, languages } from '../index.js' +import { FORMAT_VERSION, configure, diff, diffJson, expand, languages } from '../index.js' const native = createRequire(import.meta.url)('../binding.js') @@ -133,6 +133,7 @@ describe('invalid input rejects', () => { 'fractional gap length': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1, len: 1.5 }), 'gap missing len': () => expand(modified('a.rs', 'a', 'a'), { old_start: 1, new_start: 1 }), 'no gap': () => expand(modified('a.rs', 'a', 'a')), + 'diffJson null request': () => diffJson(null), } for (const [name, call] of Object.entries(bad)) { it(name, async () => { @@ -251,3 +252,128 @@ describe('diff', () => { controller.abort() }) }) + +describe('diffJson', () => { + it('resolves to the JSON diff resolves to', async () => { + const request = { files: [modified('src/view.rs', OLD_RS, NEW_RS)], syntax: true } + const json = await diffJson(request) + assert.equal(typeof json, 'string') + assert.deepEqual(JSON.parse(json), await diff(request)) + }) + + it('aborts like diff', async () => { + const controller = new AbortController() + const running = diffJson(bigRequest(64), { signal: controller.signal }) + controller.abort() + await assert.rejects(running, { name: 'AbortError' }) + }) +}) + +// The queue is process-wide; every test puts the default limit back. +describe('queue', () => { + const { max_concurrent_requests: initial } = configure() + const small = () => diff({ files: [modified('a.rs', 'a\n', 'b\n')] }) + afterEach(() => configure({ max_concurrent_requests: initial })) + + it('defaults to a quarter of the cores, at least one', () => { + assert.ok(Number.isInteger(initial) && initial >= 1) + assert.ok(initial <= Math.max(1, Math.floor(availableParallelism() / 4))) + }) + + it('runs requests in order, at most the limit at once', async () => { + assert.deepEqual(configure({ max_concurrent_requests: 1 }), { max_concurrent_requests: 1 }) + const order = [] + const done = [diff(bigRequest(64)).then(() => order.push('big'))] + for (let i = 0; i < 4; i++) done.push(small().then(() => order.push(i))) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 4 }) + await Promise.all(done) + // Unqueued, the small requests would finish long before the big one. + assert.deepEqual(order, ['big', 0, 1, 2, 3]) + assert.deepEqual(native.queue(), { limit: 1, running: 0, queued: 0 }) + }) + + it('never runs more than the limit', async () => { + configure({ max_concurrent_requests: 2 }) + let settled = false + const all = Promise.all(Array.from({ length: 6 }, () => diff(bigRequest(16)))).finally(() => { + settled = true + }) + let most = 0 + while (!settled) { + const { running, queued } = native.queue() + most = Math.max(most, running) + assert.ok(running + queued <= 6) + await new Promise((resolve) => setImmediate(resolve)) + } + await all + assert.equal(most, 2) + }) + + it('starts queued requests when the limit rises', async () => { + configure({ max_concurrent_requests: 1 }) + const done = Promise.all(Array.from({ length: 3 }, () => diff(bigRequest(16)))) + assert.equal(native.queue().queued, 2) + configure({ max_concurrent_requests: 3 }) + assert.deepEqual(native.queue(), { limit: 3, running: 3, queued: 0 }) + await done + }) + + it('rejects a request aborted while queued at once, without starting it', async () => { + configure({ max_concurrent_requests: 1 }) + const running = diff(bigRequest(64)) + const controller = new AbortController() + const reason = new Error('navigated away') + const queued = diff(bigRequest(64), { signal: controller.signal }) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 1 }) + controller.abort(reason) + assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 0 }) + let pending = true + running.finally(() => { + pending = false + }) + await assert.rejects(queued, (error) => { + assert.equal(error.name, 'AbortError') + assert.equal(error.cause, reason) + return true + }) + assert.ok(pending, 'rejected before the running request finished') + await running + }) + + it('counts time queued against budget_ms', async () => { + configure({ max_concurrent_requests: 1 }) + const running = diff(bigRequest(64)) + const doc = await diff({ files: [modified('a.rs', 'fn a() {}\n', 'fn b() {}\n')], budget_ms: 1 }) + assert.equal(doc.files[0].fallback?.kind, 'request_budget') + await running + }) + + it('does not queue expand', async () => { + configure({ max_concurrent_requests: 1 }) + let pending = true + const running = diff(bigRequest(64)).finally(() => { + pending = false + }) + const rows = await expand(modified('a.rs', 'x\n', 'x\n'), { old_start: 1, new_start: 1, len: 1 }) + assert.equal(rows.length, 1) + assert.ok(pending, 'expand finished while a diff held the only slot') + await running + }) + + for (const [name, options] of Object.entries({ + zero: { max_concurrent_requests: 0 }, + fractional: { max_concurrent_requests: 1.5 }, + 'not a number': { max_concurrent_requests: '2' }, + 'not an object': 4, + null: null, + })) { + it(`rejects a ${name} limit`, () => { + assert.throws(() => configure(options), (error) => { + assert.ok(error instanceof TypeError) + assert.equal(error.code, 'InvalidArg') + return true + }) + assert.equal(configure().max_concurrent_requests, initial) + }) + } +}) From 600279e967cfb2231e361664325b6b4a8e720932 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:00:21 +0000 Subject: [PATCH 09/16] feat(node): ship RUST_NOTICES.md with each platform package scripts/notices.mjs (npm run notices) lists the Rust crates diffy-node links on a target, with each one's license expression, repository, and the license files it ships, printing each distinct text once. The crate set comes from cargo tree, which resolves features for diffy-node alone; cargo metadata supplies package details. It fails, writing nothing, when a crate's license is not on a permissive allowlist. CI writes it into every platform package, and publishing checks it is there. --- .github/workflows/node.yml | 8 + bindings/node/.gitignore | 3 + bindings/node/README.md | 7 +- bindings/node/npm/darwin-arm64/README.md | 2 +- bindings/node/npm/darwin-arm64/package.json | 1 + bindings/node/npm/darwin-x64/README.md | 2 +- bindings/node/npm/darwin-x64/package.json | 1 + bindings/node/npm/linux-arm64-gnu/README.md | 2 +- .../node/npm/linux-arm64-gnu/package.json | 1 + bindings/node/npm/linux-x64-gnu/README.md | 2 +- bindings/node/npm/linux-x64-gnu/package.json | 1 + bindings/node/npm/win32-x64-msvc/README.md | 2 +- bindings/node/npm/win32-x64-msvc/package.json | 1 + bindings/node/package.json | 1 + bindings/node/scripts/notices.mjs | 184 ++++++++++++++++++ 15 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 bindings/node/scripts/notices.mjs diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index e234dcdd..80e6b209 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -154,6 +154,13 @@ jobs: cp "target/syntax-bundle/$TARGET/THIRD_PARTY_NOTICES.md" "$out/" ls -l "$out" + # Credits the Rust crates linked for this target, and fails on one + # whose license is not on the script's allowlist. + - name: Write the Rust notices + run: | + node bindings/node/scripts/notices.mjs --locked --target "$TARGET" --out "bindings/node/npm/$PLATFORM" + grep -c '^## ' "bindings/node/npm/$PLATFORM/RUST_NOTICES.md" + - name: Sign ad hoc (macOS) if: runner.os == 'macOS' run: codesign --force --sign - "bindings/node/npm/$PLATFORM/diffy.$PLATFORM.node" @@ -210,6 +217,7 @@ jobs: cp -R "artifacts/node-$platform/." "$dir" test -f "$dir/diffy.$platform.node" test -f "$dir/THIRD_PARTY_NOTICES.md" + test -f "$dir/RUST_NOTICES.md" done - name: Check versions against the tag diff --git a/bindings/node/.gitignore b/bindings/node/.gitignore index f85c308c..09b9643e 100644 --- a/bindings/node/.gitignore +++ b/bindings/node/.gitignore @@ -3,3 +3,6 @@ node_modules/ # Copied into each platform package at publish time. npm/*/LICENSE-* npm/*/THIRD_PARTY_NOTICES.md +npm/*/RUST_NOTICES.md +# Written by `npm run notices`. +/RUST_NOTICES.md diff --git a/bindings/node/README.md b/bindings/node/README.md index e481a594..ffc45a29 100644 --- a/bindings/node/README.md +++ b/bindings/node/README.md @@ -99,7 +99,8 @@ binary as an optional dependency: There is no musl build. Installing with `--omit=optional` leaves the binary out. Each platform package ships `THIRD_PARTY_NOTICES.md`, which lists the -grammars compiled in and their licenses. +grammars compiled in and their licenses, and `RUST_NOTICES.md`, which lists +the Rust crates compiled in with their licenses and license texts. For Electron, unpack `*.node` from the asar archive (for example with electron-builder's `asarUnpack`). @@ -120,6 +121,10 @@ The build uses the workspace's `node` profile: `release`, but with `panic = "unwind"` so that panics can be caught. Set `DIFFY_NODE_BINARY=path/to/diffy.node` to load a specific binary. +`npm run notices -- --target TRIPLE --out DIR` writes `DIR/RUST_NOTICES.md` +for the crates `diffy-node` links on that target, and fails if any has a +license outside its allowlist of permissive ones. + `.github/workflows/node.yml` builds and tests every platform. Linux builds run inside manylinux_2_28, so their binaries need glibc 2.28 at most, and CI checks this. Pushing a `node-v` tag publishes the platform diff --git a/bindings/node/npm/darwin-arm64/README.md b/bindings/node/npm/darwin-arm64/README.md index d1b2eb61..7d66cffd 100644 --- a/bindings/node/npm/darwin-arm64/README.md +++ b/bindings/node/npm/darwin-arm64/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Apple silicon. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. diff --git a/bindings/node/npm/darwin-arm64/package.json b/bindings/node/npm/darwin-arm64/package.json index b8448e74..6642c374 100644 --- a/bindings/node/npm/darwin-arm64/package.json +++ b/bindings/node/npm/darwin-arm64/package.json @@ -12,6 +12,7 @@ "files": [ "diffy.darwin-arm64.node", "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", "LICENSE-MIT", "LICENSE-APACHE" ], diff --git a/bindings/node/npm/darwin-x64/README.md b/bindings/node/npm/darwin-x64/README.md index 38c45802..41970f42 100644 --- a/bindings/node/npm/darwin-x64/README.md +++ b/bindings/node/npm/darwin-x64/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Intel. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. diff --git a/bindings/node/npm/darwin-x64/package.json b/bindings/node/npm/darwin-x64/package.json index aebca37e..1bc8415b 100644 --- a/bindings/node/npm/darwin-x64/package.json +++ b/bindings/node/npm/darwin-x64/package.json @@ -12,6 +12,7 @@ "files": [ "diffy.darwin-x64.node", "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", "LICENSE-MIT", "LICENSE-APACHE" ], diff --git a/bindings/node/npm/linux-arm64-gnu/README.md b/bindings/node/npm/linux-arm64-gnu/README.md index 3c99474d..389e4920 100644 --- a/bindings/node/npm/linux-arm64-gnu/README.md +++ b/bindings/node/npm/linux-arm64-gnu/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux arm64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. diff --git a/bindings/node/npm/linux-arm64-gnu/package.json b/bindings/node/npm/linux-arm64-gnu/package.json index 0862c3aa..adaf1043 100644 --- a/bindings/node/npm/linux-arm64-gnu/package.json +++ b/bindings/node/npm/linux-arm64-gnu/package.json @@ -12,6 +12,7 @@ "files": [ "diffy.linux-arm64-gnu.node", "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", "LICENSE-MIT", "LICENSE-APACHE" ], diff --git a/bindings/node/npm/linux-x64-gnu/README.md b/bindings/node/npm/linux-x64-gnu/README.md index 35392e9e..69145542 100644 --- a/bindings/node/npm/linux-x64-gnu/README.md +++ b/bindings/node/npm/linux-x64-gnu/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux x64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. diff --git a/bindings/node/npm/linux-x64-gnu/package.json b/bindings/node/npm/linux-x64-gnu/package.json index 1b71a702..35d1d24f 100644 --- a/bindings/node/npm/linux-x64-gnu/package.json +++ b/bindings/node/npm/linux-x64-gnu/package.json @@ -12,6 +12,7 @@ "files": [ "diffy.linux-x64-gnu.node", "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", "LICENSE-MIT", "LICENSE-APACHE" ], diff --git a/bindings/node/npm/win32-x64-msvc/README.md b/bindings/node/npm/win32-x64-msvc/README.md index 392bcd74..74b36a0c 100644 --- a/bindings/node/npm/win32-x64-msvc/README.md +++ b/bindings/node/npm/win32-x64-msvc/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Windows x64. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. +`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. diff --git a/bindings/node/npm/win32-x64-msvc/package.json b/bindings/node/npm/win32-x64-msvc/package.json index a2c4b36e..a9107c0c 100644 --- a/bindings/node/npm/win32-x64-msvc/package.json +++ b/bindings/node/npm/win32-x64-msvc/package.json @@ -12,6 +12,7 @@ "files": [ "diffy.win32-x64-msvc.node", "THIRD_PARTY_NOTICES.md", + "RUST_NOTICES.md", "LICENSE-MIT", "LICENSE-APACHE" ], diff --git a/bindings/node/package.json b/bindings/node/package.json index c8b94837..59712482 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -31,6 +31,7 @@ }, "scripts": { "build": "node scripts/build.mjs", + "notices": "node scripts/notices.mjs", "test": "node --test test/diffy.test.mjs" }, "optionalDependencies": { diff --git a/bindings/node/scripts/notices.mjs b/bindings/node/scripts/notices.mjs new file mode 100644 index 00000000..002f191c --- /dev/null +++ b/bindings/node/scripts/notices.mjs @@ -0,0 +1,184 @@ +// Writes RUST_NOTICES.md: every Rust crate compiled into the addon for a +// target, with its license and the license files it ships. Fails, writing +// nothing, if a crate's license is not on the permissive allowlist. +// +// node scripts/notices.mjs [--target TRIPLE] [--out DIR] [--locked] +// +// The grammars compiled in are credited separately, in the bundle's +// THIRD_PARTY_NOTICES.md. + +import { spawnSync } from 'node:child_process' +import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' + +const PACKAGE = 'diffy-node' +// SPDX identifiers that need no more than this notice. MPL-2.0 is +// file-level copyleft: it binds only the crate's own files, unmodified here. +const PERMITTED = new Set([ + 'MIT', + 'Apache-2.0', + 'BSD-2-Clause', + 'BSD-3-Clause', + 'ISC', + 'Unicode-3.0', + 'Zlib', + '0BSD', + 'CC0-1.0', + 'BSL-1.0', + 'MPL-2.0', +]) +const PERMITTED_EXCEPTIONS = new Set(['LLVM-exception']) +const LICENSE_FILE = /^(licen[cs]e|copying|notice)/i + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const { values } = parseArgs({ + options: { + target: { type: 'string' }, + out: { type: 'string', default: root }, + locked: { type: 'boolean', default: false }, + }, +}) +const target = values.target ?? hostTriple() +const locked = values.locked ? ['--locked'] : [] + +// The crate set comes from `cargo tree`, which resolves features for this +// package alone, as `cargo build -p` does. `cargo metadata` resolves them +// for the whole workspace, so its graph also holds crates only the app +// links (its HTTP client and TLS stack). +const tree = cargo(['tree', '-p', PACKAGE, '-e', 'normal', '--target', target, '--prefix', 'none', '--format', '{p}', ...locked]) +const used = new Set() +for (const line of tree.split('\n')) { + const match = line.match(/^(\S+) v(\S+)/) + if (match) used.add(`${match[1]} ${match[2]}`) +} +if (used.size === 0) fail(`cargo tree listed no crates for ${PACKAGE}`) + +const metadata = JSON.parse( + cargo(['metadata', '--format-version', '1', '--filter-platform', target, ...locked]), +) +const workspace = new Set(metadata.workspace_members) +const packages = new Map() +for (const pkg of metadata.packages) { + const key = `${pkg.name} ${pkg.version}` + if (!used.has(key)) continue + if (packages.has(key)) fail(`two packages are ${key}; cannot tell which is linked`) + packages.set(key, pkg) +} +for (const key of used) if (!packages.has(key)) fail(`cargo metadata has no package ${key}`) + +const crates = [...packages.values()].sort((a, b) => a.name.localeCompare(b.name) || a.version.localeCompare(b.version)) +const refused = crates.filter((pkg) => !permitted(pkg.license ?? '')) +if (refused.length > 0) { + for (const pkg of refused) { + console.error(`notices: ${pkg.name} ${pkg.version} is ${pkg.license ?? `unlicensed (license-file ${pkg.license_file})`}`) + } + fail(`${refused.length} crate(s) have a license outside the allowlist`) +} + +const ours = crates.filter((pkg) => workspace.has(pkg.id)) +const theirs = crates.filter((pkg) => !workspace.has(pkg.id)) +const out = [ + '# Rust crates in this binary', + '', + `The Rust crates compiled into \`@diffy/node\` for \`${target}\`, with their licenses and the license files each one ships. The tree-sitter grammars compiled in are listed in THIRD_PARTY_NOTICES.md.`, + '', + `Diffy's own crates (${ours.map((pkg) => `\`${pkg.name}\``).join(', ')}) are ${[...new Set(ours.map((pkg) => pkg.license))].join('; ')}: see LICENSE-MIT and LICENSE-APACHE.`, +] +// Many crates ship the same license text; it is printed once. +const printed = new Map() +for (const pkg of theirs) { + out.push('', `## ${pkg.name} ${pkg.version}`, '', `License: ${pkg.license}`) + if (pkg.repository) out.push(`Repository: ${pkg.repository}`) + const files = licenseFiles(pkg) + if (files.length === 0) out.push('', `No license file in the package; its license is \`${pkg.license}\`.`) + for (const file of files) { + const text = readFileSync(resolve(dirname(pkg.manifest_path), file), 'utf8') + const key = text.replaceAll('\r\n', '\n').replace(/[ \t]+$/gm, '').trim() + const first = printed.get(key) + if (first) { + out.push('', `${file}: the same text as ${first}.`) + continue + } + printed.set(key, `${pkg.name} ${pkg.version}'s ${file}`) + out.push('', `### ${file}`, '', ...fenced(key)) + } +} + +mkdirSync(values.out, { recursive: true }) +const output = join(values.out, 'RUST_NOTICES.md') +writeFileSync(output, `${out.join('\n')}\n`) +console.log(`${output}: ${theirs.length} crates`) + +/** The top-level license files of `pkg`, and its `license-file`. */ +function licenseFiles(pkg) { + const dir = dirname(pkg.manifest_path) + const files = new Set(readdirSync(dir).filter((name) => LICENSE_FILE.test(name) && statSync(join(dir, name)).isFile())) + if (pkg.license_file) files.add(pkg.license_file) + return [...files].sort() +} + +/** Whether an SPDX expression (or the legacy `A/B`) lets us ship the crate. */ +function permitted(expression) { + const tokens = expression.replaceAll('/', ' OR ').match(/[()]|[^\s()]+/g) ?? [] + let at = 0 + // Each side is always parsed, so `at` moves past it. + const any = () => { + let ok = all() + while (tokens[at] === 'OR') { + at++ + ok = all() || ok + } + return ok + } + const all = () => { + let ok = one() + while (tokens[at] === 'AND') { + at++ + ok = one() && ok + } + return ok + } + const one = () => { + const token = tokens[at++] + if (token === '(') { + const ok = any() + return tokens[at++] === ')' && ok + } + let ok = PERMITTED.has(token) + if (tokens[at] === 'WITH') { + at++ + ok = PERMITTED_EXCEPTIONS.has(tokens[at++]) && ok + } + return ok + } + const ok = any() + return ok && at === tokens.length +} + +/** `text` in a code fence longer than any backtick run inside it. */ +function fenced(text) { + const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length)) + const fence = '`'.repeat(Math.max(3, longest + 1)) + return [`${fence}text`, text, fence] +} + +function cargo(args) { + const run = spawnSync('cargo', args, { cwd: root, encoding: 'utf8', maxBuffer: 1 << 30, stdio: ['ignore', 'pipe', 'inherit'] }) + if (run.error) fail(`cargo: ${run.error.message}`) + if (run.status !== 0) fail(`cargo ${args[0]} failed (${run.status})`) + return run.stdout +} + +function hostTriple() { + const rustc = spawnSync('rustc', ['-vV'], { encoding: 'utf8' }) + const host = rustc.stdout?.match(/^host: (\S+)$/m)?.[1] + if (!host) fail('could not read the host triple from `rustc -vV`') + return host +} + +function fail(message) { + console.error(`notices: ${message}`) + process.exit(1) +} From 87c94eac68d6d086bc82facec547b10df435759b Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:01:02 +0000 Subject: [PATCH 10/16] fix(compare): name hunk headers by the item that contains the change Outline items carried only a start line, so the wire projection named a hunk after the nearest changed item starting above it, and a change in a top-level statement below a function got that function's name. diffy_diff::OutlineItem now holds the item's line range on its side (`lines`, from the node diffy-semantic already measures), and the header is the innermost changed item containing the hunk's first changed line with text: items on the change's side first, then the other side, where a pure insertion or deletion must fall strictly inside the item. Leading blank lines are skipped so an added or removed item names itself rather than nothing. Without a containing item the scope table fallback is unchanged (still only with syntax), and otherwise there is no header. The format gains an optional `OutlineItem.end_line` (inclusive), so FORMAT_VERSION stays 1. --- bindings/node/index.d.ts | 2 + crates/diffy-compare/FORMAT.md | 5 +- crates/diffy-compare/src/wire.rs | 101 ++++++++------ crates/diffy-compare/src/wire/tests.rs | 177 ++++++++++++++++++++++++- crates/diffy-diff/src/semantic.rs | 3 +- crates/diffy-semantic/src/align.rs | 2 +- src/app/navigation/view.rs | 2 +- tools/semantic-eval/src/main.rs | 5 +- 8 files changed, 251 insertions(+), 46 deletions(-) diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 41fb0e17..f4f41c26 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -245,6 +245,8 @@ export interface OutlineItem { side: 'old' | 'new' /** The item's first line. */ line: number + /** The item's last line, inclusive. */ + end_line?: number /** The grammar's node kind, e.g. `function_item`. */ kind: string name: string diff --git a/crates/diffy-compare/FORMAT.md b/crates/diffy-compare/FORMAT.md index fa6f4e70..e2bf0e73 100644 --- a/crates/diffy-compare/FORMAT.md +++ b/crates/diffy-compare/FORMAT.md @@ -74,7 +74,7 @@ Line-diff files still get changed-word spans on `modified` rows. | Field | Type | | | --- | --- | --- | | `old_start`, `old_len`, `new_start`, `new_len` | number | The numbers of a unified diff's `@@ -old_start,old_len +new_start,new_len @@` line. A start is the side's first line in the hunk, or the line before it when that side is empty (`0` at the top of the file). | -| `header` | string? | The item enclosing the hunk's first change, for the hunk's header line: an item name from the semantic outline (`render_preview`), or a scope label from the grammar (`fn render_preview`, `impl Parser`, `class Parser`, `parse()`). Absent when neither is known. | +| `header` | string? | The innermost item whose lines contain the hunk's first changed line (blank lines skipped), for the hunk's header line: a changed item's name from the semantic outline (`render_preview`), else a scope label from the grammar (`fn render_preview`, `impl Parser`, `class Parser`, `parse()`). Absent when nothing contains the change, as for a top-level statement between functions. | | `gap_before` | Gap? | Unchanged lines between the previous hunk (or the top of the file) and this one. | | `rows` | Row[] | In display order; each side's line numbers ascend. | @@ -171,6 +171,7 @@ independent layers: draw the change background, then the syntax colour. | --- | --- | --- | | `side` | `"old"` \| `"new"` | Which file `line` refers to: `removed` items are on the old side, the rest on the new. | | `line` | number | The item's first line. | +| `end_line` | number? | The item's last line, inclusive. | | `kind` | string | The grammar's node kind, e.g. `function_item`, `class_declaration`. | | `name` | string | | | `change` | `"added"` \| `"removed"` \| `"renamed"` \| `"moved"` \| `"edited"` | `renamed`: only identifiers inside it were renamed. | @@ -257,6 +258,7 @@ type SyntaxSpan = { start: number; end: number; class: string }; // UTF-16 type OutlineItem = { side: "old" | "new"; line: number; + end_line?: number; kind: string; name: string; change: "added" | "removed" | "renamed" | "moved" | "edited"; @@ -339,6 +341,7 @@ unit and `😀` two, so the changed `2` starts at 16 (byte 19). { "side": "new", "line": 11, + "end_line": 13, "kind": "function_item", "name": "render_preview", "change": "edited", diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs index bc50731e..8eeb6451 100644 --- a/crates/diffy-compare/src/wire.rs +++ b/crates/diffy-compare/src/wire.rs @@ -333,6 +333,9 @@ pub struct OutlineItem { pub side: Side, /// One-based first line of the item on `side`. pub line: u32, + /// One-based last line of the item on `side`, inclusive. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_line: Option, /// The grammar's item kind, such as `function_item`. pub kind: String, pub name: String, @@ -515,7 +518,8 @@ fn outline_item(item: &diffy_diff::OutlineItem) -> OutlineItem { }; OutlineItem { side: item.side.into(), - line: item.line + 1, + line: item.lines.start + 1, + end_line: Some(item.lines.end), kind: item.kind.to_string(), name: item.name.to_string(), change, @@ -731,9 +735,9 @@ impl<'a> Projector<'a> { } } - /// The item enclosing the hunk's first change: the nearest changed - /// item of the semantic outline starting at or before it, else the - /// innermost scope of the grammar's scope table. + /// The innermost item whose lines contain the hunk's first changed + /// line with text: a changed item of the semantic outline, else a + /// scope of the grammar's scope table. `None` when nothing contains it. fn header(&self, hunk: u32) -> Option { let (h, b) = (self.doc.hunks(), self.doc.blocks()); let hi = hunk as usize; @@ -741,46 +745,69 @@ impl<'a> Projector<'a> { .clone() .map(|block| block as usize) .find(|&bi| b.kind[bi] == BlockKind::Change); - // Zero-based anchors on each side, and the side the change shows. - let (old, new, side) = match first_change { - Some(bi) => ( - b.old_line[bi] - 1, - b.new_line[bi] - 1, - if b.new_len[bi] > 0 { - DocSide::New - } else { - DocSide::Old - }, - ), + // Zero-based first line of the change on each side, and whether + // the side has changed lines there; a side without them holds the + // change between the anchor line and the one before. + let (old, new) = match first_change { + Some(bi) => { + // Blank lines lead most added and removed items and belong + // to the scope around them, so the first one with text + // anchors the change. + let first_text = |side: DocSide, store: u32, len: u32| { + (0..len) + .find(|&k| { + self.display(side, store + k) + .bytes() + .any(|byte| !byte.is_ascii_whitespace()) + }) + .unwrap_or(0) + }; + let old_skip = first_text(DocSide::Old, b.old_store[bi], b.old_len[bi]); + let new_skip = first_text(DocSide::New, b.new_store[bi], b.new_len[bi]); + ( + (b.old_line[bi] - 1 + old_skip, b.old_len[bi] > 0), + (b.new_line[bi] - 1 + new_skip, b.new_len[bi] > 0), + ) + } None => ( - h.old_start[hi].saturating_sub(1), - h.new_start[hi].saturating_sub(1), - DocSide::New, + (h.old_start[hi].saturating_sub(1), false), + (h.new_start[hi].saturating_sub(1), true), ), }; + // The change shows on the new side when it has lines there. + let (side, other) = if new.1 { + (DocSide::New, DocSide::Old) + } else { + (DocSide::Old, DocSide::New) + }; + let anchor = |side: DocSide| match side { + DocSide::Old => old, + DocSide::New => new, + }; if let Some(semantics) = self.doc.semantics(0) { - // Items list no end line, so the nearest start before the - // change stands in for the enclosing item. Edited items sit on - // the new side, removed ones on the old; ties go to new. - let nearest = semantics - .outline() - .iter() - .filter_map(|item| { - let anchor = match item.side { - DocSide::Old => old, - DocSide::New => new, - }; - (item.line <= anchor).then(|| (anchor - item.line, item.side, item)) - }) - .min_by_key(|&(distance, side, _)| (distance, side == DocSide::Old)); - if let Some((_, _, item)) = nearest { + // Edited items sit on the new side and removed ones on the + // old, so a side with no containing item defers to the other. + // Items on one side nest, so the shortest is the innermost. + let innermost = |side: DocSide| { + let (line, lines) = anchor(side); + semantics + .outline() + .iter() + .filter(|item| item.side == side) + .filter(|item| { + if lines { + item.lines.contains(&line) + } else { + item.lines.start < line && line < item.lines.end + } + }) + .min_by_key(|item| item.lines.len()) + }; + if let Some(item) = innermost(side).or_else(|| innermost(other)) { return Some(item.name.to_string()); } } - let line = match side { - DocSide::Old => old, - DocSide::New => new, - }; + let (line, _) = anchor(side); // Sorted by start, longer first, so the last span containing the // line is the innermost. self.scopes(side) diff --git a/crates/diffy-compare/src/wire/tests.rs b/crates/diffy-compare/src/wire/tests.rs index 7a86ab15..d8d867ea 100644 --- a/crates/diffy-compare/src/wire/tests.rs +++ b/crates/diffy-compare/src/wire/tests.rs @@ -290,7 +290,7 @@ fn semantic_diff() -> (FileInput, FileDiff) { file_builder .outline(vec![diffy_diff::OutlineItem { side: DocSide::New, - line: 1, + lines: 1..4, kind: Arc::from("function_item"), name: Arc::from("render_preview"), change: ItemChange::Edited(2), @@ -328,6 +328,7 @@ fn semantic_documents_keep_spans_outline_moves_and_renames() { [OutlineItem { side: Side::New, line: 2, + end_line: Some(4), kind: "function_item".into(), name: "render_preview".into(), change: ItemChangeKind::Edited, @@ -379,7 +380,7 @@ fn semantic_documents_keep_spans_outline_moves_and_renames() { } #[test] -fn hunk_header_needs_an_item_before_the_change() { +fn hunk_header_needs_an_item_containing_the_change() { let (input, diff) = semantic_diff(); let mut document = DocumentBuilder::new(); let source = diff.document.text(0, DocSide::Old).clone(); @@ -393,7 +394,7 @@ fn hunk_header_needs_an_item_before_the_change() { .context(1) .outline(vec![diffy_diff::OutlineItem { side: DocSide::Old, - line: 3, + lines: 3..4, kind: Arc::from("function_item"), name: Arc::from("later"), change: ItemChange::Removed, @@ -407,6 +408,70 @@ fn hunk_header_needs_an_item_before_the_change() { assert_eq!(file.hunks[0].header, None); } +#[test] +fn hunk_headers_name_the_innermost_containing_item() { + let item = |side, lines, name: &str| diffy_diff::OutlineItem { + side, + lines, + kind: Arc::from("function_item"), + name: Arc::from(name), + change: ItemChange::Edited(1), + }; + let header = |outline: Vec, (old_len, new_len): (u32, u32)| { + let side = |len, line| match len { + 0 => "zero\none\nthree\n".to_owned(), + _ => format!("zero\none\n{line}\nthree\n"), + }; + let (old, new) = (side(old_len, "old"), side(new_len, "new")); + let mut document = DocumentBuilder::new(); + let mut file_builder = document.file( + meta(), + SourceCoverage::Full, + TextStore::new(old.as_str()), + TextStore::new(new.as_str()), + ); + file_builder + .structural(true) + .hunk(0, 0) + .context(2) + .change(old_len, new_len) + .context(1) + .outline(outline); + file_builder.finish().expect("builds"); + let diff = FileDiff { + document: document.build(), + fallback: None, + }; + file(&input(&old, &new), &diff, &WireOptions::default()).hunks[0] + .header + .clone() + }; + // The change is on line 3 (zero-based 2) of each side. + let nested = || { + vec![ + item(DocSide::New, 0..4, "outer"), + item(DocSide::New, 1..4, "inner"), + item(DocSide::Old, 2..3, "removed"), + ] + }; + assert_eq!(header(nested(), (1, 1)).as_deref(), Some("inner")); + // Only the old side has lines, so its items come first. + assert_eq!(header(nested(), (1, 0)).as_deref(), Some("removed")); + // An old-only change without an old item takes the new item around + // the gap it leaves, which must hold the lines on both sides of it. + let new_only = || { + vec![ + item(DocSide::New, 0..4, "outer"), + item(DocSide::New, 2..4, "after"), + ] + }; + assert_eq!(header(new_only(), (1, 0)).as_deref(), Some("outer")); + assert_eq!( + header(vec![item(DocSide::New, 0..2, "before")], (1, 0)), + None + ); +} + #[test] fn aligned_rows_pair_lines_where_the_semantic_diff_put_them() { let old = "keep\none\ntwo\nend\n"; @@ -770,3 +835,109 @@ fn files_projected_past_the_deadline_skip_syntax() { assert!(!file.hunks.is_empty()); } } + +/// The hunk headers of a real semantic diff of `old` to `new`, without +/// syntax, so they come from the outline alone; `None` without the pack. +fn outline_headers(language: &str, path: &str, old: &str, new: &str) -> Option> { + let store = diffy_syntax::testing::store_with(language)?; + let input = FileInput { + status: FileStatus::Modified, + old_path: Some(path.into()), + new_path: Some(path.into()), + old: old.as_bytes().to_vec(), + new: new.as_bytes().to_vec(), + }; + let diff = crate::diff_file(&input, &crate::Options::new(&store)).expect("diffs"); + assert_eq!(diff.fallback, None); + let file = file(&input, &diff, &WireOptions::default()); + Some( + file.hunks + .into_iter() + .map(|hunk| hunk.header.unwrap_or_default()) + .collect(), + ) +} + +#[test] +fn hunk_headers_name_only_items_containing_the_change() { + let old = "def first(): + a = 1 + b = 2 + c = 3 + return a + + +x = 1 +y = 2 +z = 3 +w = 4 +v = 5 +u = 6 +t = 7 +s = 8 +r = 9 + + +def second(): + p = 1 + q = 2 + return p +"; + let new = old + .replace("a = 1", "a = 10") + .replace("v = 5", "v = 50") + .replace("q = 2", "q = 20"); + let Some(headers) = outline_headers("python", "src/app.py", old, &new) else { + return; + }; + // The middle change is a top-level statement below `first`. + assert_eq!(headers, ["first", "", "second"]); +} + +#[test] +fn hunk_headers_pick_the_innermost_item_on_the_change_side() { + let old = "impl Parser { + fn render(&self) { + draw(1); + draw(2); + draw(3); + draw(4); + draw(5); + draw(6); + } + + fn other(&self) { + a(); + b(); + c(); + d(); + e(); + f(); + g(); + h(); + } + + fn stay(&self) { + one(); + two(); + three(); + four(); + } + + fn removed(&self) { + gone(); + } +} +"; + let new = old + .replace("draw(1)", "draw(10)") + .replace(" b();\n", "") + .replace("\n fn removed(&self) {\n gone();\n }\n", ""); + let Some(headers) = outline_headers("rust", "src/lib.rs", old, &new) else { + return; + }; + // An edit inside a method names the method, not its impl; a deletion + // inside an edited method names it from the new side; a removed + // method names itself from the old side. + assert_eq!(headers, ["render", "other", "removed"]); +} diff --git a/crates/diffy-diff/src/semantic.rs b/crates/diffy-diff/src/semantic.rs index 054b9705..5ace8266 100644 --- a/crates/diffy-diff/src/semantic.rs +++ b/crates/diffy-diff/src/semantic.rs @@ -116,7 +116,8 @@ pub struct Rename { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OutlineItem { pub side: Side, - pub line: u32, + /// Zero-based lines the item spans on `side`; never empty. + pub lines: Range, pub kind: Arc, pub name: Arc, pub change: ItemChange, diff --git a/crates/diffy-semantic/src/align.rs b/crates/diffy-semantic/src/align.rs index 58fb0881..62798d7d 100644 --- a/crates/diffy-semantic/src/align.rs +++ b/crates/diffy-semantic/src/align.rs @@ -680,7 +680,7 @@ fn outline(sides: &[Side<'_, '_>; 2], matching: &Matching) -> Vec { }; out.push(OutlineItem { side: which, - line: item.lines.start, + lines: item.lines.clone(), kind: item.kind.into(), name: side.tree.text(item.name.clone()).into(), change, diff --git a/src/app/navigation/view.rs b/src/app/navigation/view.rs index 917327ff..c5cf31f6 100644 --- a/src/app/navigation/view.rs +++ b/src/app/navigation/view.rs @@ -729,7 +729,7 @@ fn item_row(file: &ManifestFile, index: u32, tree: bool, theme: &Theme) -> AnyEl file: file.key, revision: file.revision, side: item.side, - lines: item.line..item.line + 1, + lines: item.lines.start..item.lines.start + 1, }; let mono = TextStyle::MonoSmall.font_size(theme); let small = TextStyle::BodySmall.font_size(theme); diff --git a/tools/semantic-eval/src/main.rs b/tools/semantic-eval/src/main.rs index 64aa0a33..46ce05d4 100644 --- a/tools/semantic-eval/src/main.rs +++ b/tools/semantic-eval/src/main.rs @@ -154,9 +154,10 @@ fn show(args: ShowArgs) -> Result<(), String> { if let Some(semantics) = run.doc.semantics(0) { for item in semantics.outline() { println!( - "outline {:?} {} {} {} {:?}", + "outline {:?} {}-{} {} {} {:?}", item.side, - item.line + 1, + item.lines.start + 1, + item.lines.end, item.kind, item.name, item.change From 8c1d9469ecf3dd7a8ad27dcdd384f2b3a9ef41aa Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:01:43 +0000 Subject: [PATCH 11/16] fix(node): reproduce napi-rs's MIT notice, which its crates do not ship --- bindings/node/notices/napi-rs-LICENSE | 43 +++++++++++++++++++++++++++ bindings/node/scripts/notices.mjs | 13 ++++++-- 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 bindings/node/notices/napi-rs-LICENSE diff --git a/bindings/node/notices/napi-rs-LICENSE b/bindings/node/notices/napi-rs-LICENSE new file mode 100644 index 00000000..7fe7e35e --- /dev/null +++ b/bindings/node/notices/napi-rs-LICENSE @@ -0,0 +1,43 @@ +MIT License + +Copyright (c) 2020-present LongYinan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +MIT License + +Copyright (c) 2018 GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/node/scripts/notices.mjs b/bindings/node/scripts/notices.mjs index 002f191c..224962be 100644 --- a/bindings/node/scripts/notices.mjs +++ b/bindings/node/scripts/notices.mjs @@ -31,6 +31,10 @@ const PERMITTED = new Set([ ]) const PERMITTED_EXCEPTIONS = new Set(['LLVM-exception']) const LICENSE_FILE = /^(licen[cs]e|copying|notice)/i +// Crates whose packages ship no license file, with the text from their +// repository, so their copyright notice is still reproduced. +const NAPI_RS = { path: resolve(dirname(fileURLToPath(import.meta.url)), '../notices/napi-rs-LICENSE'), source: 'github.com/napi-rs/napi-rs' } +const FALLBACK_LICENSES = new Map(['napi', 'napi-sys', 'napi-derive', 'napi-derive-backend'].map((name) => [name, NAPI_RS])) const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') const { values } = parseArgs({ @@ -92,9 +96,12 @@ for (const pkg of theirs) { out.push('', `## ${pkg.name} ${pkg.version}`, '', `License: ${pkg.license}`) if (pkg.repository) out.push(`Repository: ${pkg.repository}`) const files = licenseFiles(pkg) - if (files.length === 0) out.push('', `No license file in the package; its license is \`${pkg.license}\`.`) - for (const file of files) { - const text = readFileSync(resolve(dirname(pkg.manifest_path), file), 'utf8') + const fallback = FALLBACK_LICENSES.get(pkg.name) + if (files.length === 0 && !fallback) out.push('', `No license file in the package; its license is \`${pkg.license}\`.`) + const paths = files.map((file) => [file, resolve(dirname(pkg.manifest_path), file)]) + if (files.length === 0 && fallback) paths.push([`LICENSE (from ${fallback.source})`, fallback.path]) + for (const [file, path] of paths) { + const text = readFileSync(path, 'utf8') const key = text.replaceAll('\r\n', '\n').replace(/[ \t]+$/gm, '').trim() const first = printed.get(key) if (first) { From 6bcec5e1a7e91402d4bfb6221df593ae6c8390a0 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:07:27 +0000 Subject: [PATCH 12/16] refactor(syntax): split the tree-sitter runtime from pack loading Add a `runtime` feature (tree-sitter, highlighting, injections, scopes, symbols, GrammarStore/StoreConfig) under `engine`, which now adds only local pack loading (libloading, ring, serde/serde_json manifests, thiserror). `bundled` depends on `runtime`, so a bundled-only build links no pack loader and has no `StoreConfig::local_packs`. Grammar compilation reports a crate-private GrammarError that converts into the same PackError variants, so pack errors are unchanged. diffy-compare needs only `runtime` (its tests take `engine` for `diffy_syntax::testing`); the app, diffy-ui and syntax-pack keep `engine`/`download` and their behaviour. @diffy/node drops libloading 0.8, ring, untrusted, getrandom and thiserror: the stripped linux-arm64 addon goes from 21,694,360 to 21,577,000 bytes (-115 KiB). --- bindings/node/Cargo.toml | 2 +- crates/diffy-compare/Cargo.toml | 7 +- crates/diffy-syntax/Cargo.toml | 15 +-- crates/diffy-syntax/PACKS.md | 3 + crates/diffy-syntax/README.md | 11 ++- crates/diffy-syntax/src/engine.rs | 56 +++++++++-- crates/diffy-syntax/src/lib.rs | 22 +++-- crates/diffy-syntax/src/navigate.rs | 10 +- crates/diffy-syntax/src/pack.rs | 17 +++- crates/diffy-syntax/src/scope.rs | 40 ++++---- crates/diffy-syntax/src/store.rs | 139 ++++++++++++++++------------ crates/diffy-syntax/src/symbols.rs | 22 ++--- 12 files changed, 213 insertions(+), 131 deletions(-) diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 9f30cd9d..c91aaec6 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -24,7 +24,7 @@ napi = { version = "3", default-features = false, features = ["napi4", "dyn-symb # The `#[napi]` attribute. Without `type-def`: index.d.ts is written by # hand from FORMAT.md, which generated types cannot express. napi-derive = { version = "3", default-features = false, features = ["strict"] } -# Results cross into JS as one JSON string; already built by diffy-syntax. +# Results cross into JS as one JSON string. serde_json = "1.0" [build-dependencies] diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml index b75f2fd0..c97619f6 100644 --- a/crates/diffy-compare/Cargo.toml +++ b/crates/diffy-compare/Cargo.toml @@ -12,14 +12,13 @@ publish = false [dependencies] diffy-diff = { path = "../diffy-diff" } diffy-semantic = { path = "../diffy-semantic" } -diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } +diffy-syntax = { path = "../diffy-syntax", features = ["runtime"] } # The line diff gix runs (histogram plus Git's slider heuristics), the same # crate and version gix already pulls into the app, so line diffs match Git # and the app links one copy. imara-diff = { package = "gix-imara-diff", version = "0.2.1", default-features = false } tracing = "0.1.44" -# Only for `wire`; diffy-syntax's engine already builds serde, so the -# feature adds no new crate. +# Only for `wire`. serde = { version = "1.0", default-features = false, features = ["derive", "std"], optional = true } [features] @@ -29,3 +28,5 @@ serde = ["dep:serde"] [dev-dependencies] serde_json = "1.0" +# Tests load grammar packs through `diffy_syntax::testing`. +diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } diff --git a/crates/diffy-syntax/Cargo.toml b/crates/diffy-syntax/Cargo.toml index dd49defd..b90777eb 100644 --- a/crates/diffy-syntax/Cargo.toml +++ b/crates/diffy-syntax/Cargo.toml @@ -15,6 +15,7 @@ publish = false [dependencies] tree-sitter = { version = "0.26", optional = true } tree-sitter-language = { version = "0.1", optional = true } +tracing = { version = "0.1.44", optional = true } libloading = { version = "0.8", optional = true } # SHA-256 of pack files, and Ed25519 for the index with `download`. The # same ring that ureq's rustls backend builds. @@ -22,7 +23,6 @@ ring = { version = "0.17", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } serde_json = { version = "1.0", optional = true } thiserror = { version = "2", optional = true } -tracing = { version = "0.1.44", optional = true } # Resumable, SHA-checked downloads, the trusted key type, and the HTTP # client (ureq with rustls), shared with the updater. quark-update = { workspace = true, optional = true } @@ -30,22 +30,25 @@ ureq = { version = "3", default-features = false, features = ["rustls"], optiona [features] default = [] -# The tree-sitter runtime and pack loading from local directories. +# The tree-sitter runtime: highlighting, injections, scopes, and symbols +# over the grammars a store gets from `bundled` or, with `engine`, packs. +runtime = ["dep:tree-sitter", "dep:tree-sitter-language", "dep:tracing"] +# Pack loading from local directories: manifests, SHA-256 checks, and +# shared libraries. engine = [ - "dep:tree-sitter", - "dep:tree-sitter-language", + "runtime", "dep:libloading", "dep:ring", "dep:serde", "dep:serde_json", "dep:thiserror", - "dep:tracing", ] # Downloading packs on demand from a signed index. download = ["engine", "dep:quark-update", "dep:ureq"] # Grammars compiled in from the bundle `syntax-pack bundle` writes, named # by $DIFFY_SYNTAX_BUNDLE at build time (empty, with a warning, without it). -bundled = ["engine"] +# Needs no pack loading. +bundled = ["runtime"] [dev-dependencies] tempfile = "3" diff --git a/crates/diffy-syntax/PACKS.md b/crates/diffy-syntax/PACKS.md index 3ac58ad4..35e3e1ee 100644 --- a/crates/diffy-syntax/PACKS.md +++ b/crates/diffy-syntax/PACKS.md @@ -151,6 +151,9 @@ as a headless server. The Diffy app does not use it. links it. - Without `DIFFY_SYNTAX_BUNDLE` the feature builds with an empty bundle and a cargo warning. +- `bundled` needs only the `runtime` tier, so a bundled-only build links no + pack loader (`libloading`, `ring`, JSON manifests) and has no + `StoreConfig::local_packs`. Turn on `engine` too to also load packs. - Bundled grammars get the same tags, queries, ABI check, and `catalog()` entries as packs; nothing is loaded or hashed at run time. - `diffy_syntax::bundled::languages()` lists what a build has. diff --git a/crates/diffy-syntax/README.md b/crates/diffy-syntax/README.md index 9d4760ba..5925ce9c 100644 --- a/crates/diffy-syntax/README.md +++ b/crates/diffy-syntax/README.md @@ -7,10 +7,13 @@ pack directories the app trusts or, with the `download` feature, from a signed index. Code renders plain until its grammar is available. Builds that cannot load packs (a headless server) link a fixed set with `bundled`. -Features: `engine` (the tree-sitter runtime and local packs), `download` -(fetching packs over HTTPS with Ed25519-verified indexes), and `bundled` -(grammars compiled in from `syntax-pack bundle`). Without `engine` the crate -has no dependencies and every lookup falls back to plain text. +Features: `runtime` (the tree-sitter runtime: highlighting, injections, +scopes, symbols), `engine` (`runtime` plus local packs), `download` +(`engine` plus fetching packs over HTTPS with Ed25519-verified indexes), and +`bundled` (`runtime` plus grammars compiled in from `syntax-pack bundle`, +with no pack loading, `libloading`, `ring`, or JSON manifests). Without +`runtime` the crate has no dependencies and every lookup falls back to +plain text. `highlight` runs synchronously; `HighlightWorker` runs on a background thread and drops requests a newer generation superseded, which suits code diff --git a/crates/diffy-syntax/src/engine.rs b/crates/diffy-syntax/src/engine.rs index f292d32f..380b7141 100644 --- a/crates/diffy-syntax/src/engine.rs +++ b/crates/diffy-syntax/src/engine.rs @@ -1,5 +1,6 @@ -//! Loading grammars from packs, parsing, and query evaluation, including -//! languages embedded in others (tree-sitter injections). +//! Loading grammars from packs or the bundle, parsing, and query +//! evaluation, including languages embedded in others (tree-sitter +//! injections). //! //! A [`Grammar`] is shared by every thread; parsers are not `Sync`, so each //! thread keeps its own per grammar. @@ -13,6 +14,7 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; use std::ops::ControlFlow; +#[cfg(feature = "engine")] use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -20,10 +22,43 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tree_sitter as ts; use tree_sitter::StreamingIterator; -use crate::pack::{PackError, PackManifest, supported_abi, verify_files}; +#[cfg(feature = "engine")] +use crate::pack::{PackError, PackManifest, verify_files}; use crate::store::Tag; use crate::{HighlightKind, HighlightSpan, LanguageId}; +/// Grammar ABI versions (tree-sitter `LANGUAGE_VERSION`) this runtime can +/// load. +pub fn supported_abi() -> std::ops::RangeInclusive { + ts::MIN_COMPATIBLE_LANGUAGE_VERSION as u32..=ts::LANGUAGE_VERSION as u32 +} + +/// Why a grammar, from a pack or the bundle, did not compile. Packs report +/// it as the matching `pack::PackError`. +#[derive(Debug)] +pub(crate) enum GrammarError { + Abi { abi: u32, min: u32, max: u32 }, + Query(String), + InjectionQuery(String), +} + +impl std::fmt::Display for GrammarError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Abi { abi, min, max } => { + write!( + f, + "grammar ABI {abi} is outside the supported {min}..={max}" + ) + } + Self::Query(error) => write!(f, "the highlight query does not compile: {error}"), + Self::InjectionQuery(error) => { + write!(f, "the injection query does not compile: {error}") + } + } + } +} + /// A loaded language: its tree-sitter grammar, compiled highlight query, /// and compiled injection query when its pack has one. pub(crate) struct Grammar { @@ -59,6 +94,7 @@ impl Grammar { /// The library stays loaded for the rest of the process: tree-sitter /// languages, parsers, and trees on other threads point into it, and /// nothing tracks when the last of them is gone. + #[cfg(feature = "engine")] pub(crate) fn load(dir: &Path, manifest: &PackManifest) -> Result { verify_files(dir, manifest)?; let highlights = std::fs::read_to_string(dir.join(&manifest.highlights.path))?; @@ -97,7 +133,7 @@ impl Grammar { /// A grammar compiled into the binary, with its queries. #[cfg(feature = "bundled")] - pub(crate) fn bundled(bundled: &crate::bundled::Bundled) -> Result { + pub(crate) fn bundled(bundled: &crate::bundled::Bundled) -> Result { // SAFETY: the function is the tree-sitter language function of a // grammar `syntax-pack bundle` compiled from pinned, hashed sources // and linked into this binary. @@ -155,20 +191,20 @@ impl Grammar { expected_abi: u32, highlights: &str, injections: Option<&str>, - ) -> Result { + ) -> Result { let language = ts::Language::new(language); // The declared ABI was checked; this checks the grammar agrees. let abi = language.abi_version() as u32; if expected_abi != abi || !supported_abi().contains(&abi) { let range = supported_abi(); - return Err(PackError::Abi { + return Err(GrammarError::Abi { abi, min: *range.start(), max: *range.end(), }); } let query = - lenient_query(&language, highlights).map_err(|e| PackError::Query(e.to_string()))?; + lenient_query(&language, highlights).map_err(|e| GrammarError::Query(e.to_string()))?; let capture_kinds = query .capture_names() .iter() @@ -360,9 +396,9 @@ struct InjectionPattern { } impl Injections { - fn compile(language: &ts::Language, text: &str) -> Result { - let query = - lenient_query(language, text).map_err(|e| PackError::InjectionQuery(e.to_string()))?; + fn compile(language: &ts::Language, text: &str) -> Result { + let query = lenient_query(language, text) + .map_err(|e| GrammarError::InjectionQuery(e.to_string()))?; let patterns = (0..query.pattern_count()) .map(|index| { let mut pattern = InjectionPattern::default(); diff --git a/crates/diffy-syntax/src/lib.rs b/crates/diffy-syntax/src/lib.rs index c25f10db..b87c9ccb 100644 --- a/crates/diffy-syntax/src/lib.rs +++ b/crates/diffy-syntax/src/lib.rs @@ -27,9 +27,11 @@ //! an embedded grammar downloads, its region keeps the host's colors, and //! the worker sends a newer revision of the result when it arrives. //! -//! Features: `engine` (the tree-sitter runtime and local packs), -//! `download` (fetching packs), and `bundled` (grammars linked in). Without -//! `engine` every lookup is plain and the crate has no dependencies. +//! Features: `runtime` (the tree-sitter runtime: highlighting, injections, +//! scopes, and symbols), `engine` (`runtime` plus loading local packs), +//! `download` (`engine` plus fetching packs), and `bundled` (`runtime` plus +//! grammars linked in, with no pack loading). Without `runtime` every +//! lookup is plain and the crate has no dependencies. //! //! # Threat model //! @@ -63,11 +65,19 @@ //! hashed sources, so they are as trusted as the rest of it; nothing is //! loaded or hashed at run time. +// `runtime` alone has no grammar source: it is for libraries that take a +// store from a crate turning on `bundled` or `engine`, which Cargo unifies +// in. Until then the code only a source reaches is dead. +#![cfg_attr( + all(feature = "runtime", not(any(feature = "engine", feature = "bundled"))), + allow(dead_code, unused_mut, unused_variables) +)] + #[cfg(feature = "bundled")] pub mod bundled; #[cfg(feature = "download")] mod download; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] mod engine; mod navigate; #[cfg(feature = "engine")] @@ -89,10 +99,10 @@ pub use navigate::{DefinitionLookup, SourcePos}; pub use quark_update::PublicKey; pub use scope::{ScopeResult, ScopeSpan, ScopeTable, ScopeWorker, scope_table}; pub use store::{Catalog, GrammarState, GrammarStatus, GrammarStore, LanguageStatus}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub use store::{GrammarQueries, Injector, StoreConfig}; pub use symbols::{Symbol, SymbolKind}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub use symbols::{Tagger, TaggerLookup}; pub use worker::{ HighlightFocus, HighlightPart, HighlightRequest, HighlightWorker, Highlighted, Priority, diff --git a/crates/diffy-syntax/src/navigate.rs b/crates/diffy-syntax/src/navigate.rs index d02f08a5..bb9c28f6 100644 --- a/crates/diffy-syntax/src/navigate.rs +++ b/crates/diffy-syntax/src/navigate.rs @@ -15,14 +15,14 @@ pub struct DefinitionLookup { pub symbols: Vec, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct LocalsQuery { query: tree_sitter::Query, scope: Option, definitions: Vec>, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl LocalsQuery { pub(crate) fn compile(grammar: &tree_sitter::Language, source: &str) -> Option { let query = match tree_sitter::Query::new(grammar, source) { @@ -123,7 +123,7 @@ impl LocalsQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn line_start(source: &str, line: u32) -> Option { if line == 0 { return Some(0); @@ -134,7 +134,7 @@ fn line_start(source: &str, line: u32) -> Option { .map(|(index, _)| index + 1) } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn word_node(tree: &tree_sitter::Tree, byte: usize) -> Option> { let root = tree.root_node(); let leaf = |at: usize| { @@ -145,7 +145,7 @@ fn word_node(tree: &tree_sitter::Tree, byte: usize) -> Option std::ops::RangeInclusive { - tree_sitter::MIN_COMPATIBLE_LANGUAGE_VERSION as u32..=tree_sitter::LANGUAGE_VERSION as u32 -} +pub use crate::engine::supported_abi; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PackManifest { @@ -145,6 +141,17 @@ pub enum PackError { InjectionQuery(String), } +impl From for PackError { + fn from(error: crate::engine::GrammarError) -> Self { + use crate::engine::GrammarError; + match error { + GrammarError::Abi { abi, min, max } => Self::Abi { abi, min, max }, + GrammarError::Query(error) => Self::Query(error), + GrammarError::InjectionQuery(error) => Self::InjectionQuery(error), + } + } +} + impl PackManifest { /// Checks every field the loader acts on, for packs built for /// `target`. Run before a manifest's paths or symbol are used. diff --git a/crates/diffy-syntax/src/scope.rs b/crates/diffy-syntax/src/scope.rs index b0bed63e..81abd291 100644 --- a/crates/diffy-syntax/src/scope.rs +++ b/crates/diffy-syntax/src/scope.rs @@ -8,7 +8,7 @@ use crate::{GrammarStore, LanguageId}; const CACHE_LIMIT: usize = 128; const SOURCE_LIMIT: usize = 16 << 20; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const LABEL_LIMIT: usize = 48; #[derive(Debug, Clone, PartialEq, Eq)] @@ -39,7 +39,7 @@ impl ScopeTable { .collect() } - #[cfg_attr(not(feature = "engine"), allow(dead_code))] + #[cfg_attr(not(feature = "runtime"), allow(dead_code))] fn from_spans(mut spans: Vec) -> Self { spans.sort_by(|a, b| { a.lines @@ -61,7 +61,7 @@ fn scope_table_pending( language: &LanguageId, source: &str, ) -> (ScopeTable, bool) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { if source.is_empty() || source.len() > SOURCE_LIMIT { return (ScopeTable::default(), false); @@ -84,14 +84,14 @@ fn scope_table_pending( crate::store::Tag::Unavailable => (ScopeTable::default(), false), } } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (store, language, source, SOURCE_LIMIT); (ScopeTable::default(), false) } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn scope_language(tag: &str) -> &str { match tag { "rs" => "rust", @@ -104,14 +104,14 @@ fn scope_language(tag: &str) -> &str { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] struct Pattern { query: &'static str, prefix: &'static str, suffix: &'static str, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const fn pattern(query: &'static str, prefix: &'static str, suffix: &'static str) -> Pattern { Pattern { query, @@ -120,7 +120,7 @@ const fn pattern(query: &'static str, prefix: &'static str, suffix: &'static str } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const RUST: &[Pattern] = &[ pattern( "(mod_item name: (identifier) @name body: (_)) @scope", @@ -139,7 +139,7 @@ const RUST: &[Pattern] = &[ pattern("(function_item name: (_) @name) @scope", "fn ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const ECMA: &[Pattern] = &[ pattern("(class_declaration name: (_) @name) @scope", "class ", ""), pattern( @@ -172,7 +172,7 @@ const ECMA: &[Pattern] = &[ ), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const GO: &[Pattern] = &[ pattern("(type_spec name: (_) @name) @scope", "type ", ""), pattern( @@ -183,13 +183,13 @@ const GO: &[Pattern] = &[ pattern("(function_declaration name: (_) @name) @scope", "func ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const PYTHON: &[Pattern] = &[ pattern("(class_definition name: (_) @name) @scope", "class ", ""), pattern("(function_definition name: (_) @name) @scope", "def ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const C: &[Pattern] = &[ pattern( "(namespace_definition name: (_) @name) @scope", @@ -228,7 +228,7 @@ const C: &[Pattern] = &[ ), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const ZIG: &[Pattern] = &[ pattern( "(variable_declaration (identifier) @name [(struct_declaration) (enum_declaration) (union_declaration)]) @scope", @@ -239,14 +239,14 @@ const ZIG: &[Pattern] = &[ pattern("(test_declaration (_) @name) @scope", "test ", ""), ]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] const NIX: &[Pattern] = &[pattern( "(binding attrpath: (_) @name expression: [(function_expression) (attrset_expression) (rec_attrset_expression) (let_expression)]) @scope", "", "", )]; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn patterns(language: &str) -> &'static [Pattern] { match language { "rust" => RUST, @@ -260,7 +260,7 @@ fn patterns(language: &str) -> &'static [Pattern] { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct ScopeQuery { query: tree_sitter::Query, labels: Vec<(&'static str, &'static str)>, @@ -270,7 +270,7 @@ pub(crate) struct ScopeQuery { receiver: Option, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl ScopeQuery { pub(crate) fn compile(grammar: &tree_sitter::Language, language: &str) -> Option { let usable: Vec<&Pattern> = patterns(language) @@ -334,7 +334,7 @@ impl ScopeQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn clean(text: &str) -> String { let mut out = String::new(); for word in text.split_whitespace() { @@ -456,9 +456,9 @@ impl Drop for ScopeWorker { } fn tracing_warn(error: &std::io::Error) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] tracing::warn!(%error, "scope worker unavailable"); - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = error; } diff --git a/crates/diffy-syntax/src/store.rs b/crates/diffy-syntax/src/store.rs index 2484cfd8..79b5668e 100644 --- a/crates/diffy-syntax/src/store.rs +++ b/crates/diffy-syntax/src/store.rs @@ -1,13 +1,13 @@ //! Where grammars come from: [`GrammarStore`]. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::collections::HashMap; #[cfg(feature = "engine")] use std::path::{Path, PathBuf}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::sync::{Arc, Mutex, MutexGuard}; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use crate::engine::Grammar; #[cfg(feature = "engine")] use crate::pack::{self, PackManifest}; @@ -27,7 +27,7 @@ pub struct GrammarStatus { pub state: GrammarState, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Debug, Clone)] pub struct GrammarQueries { pub language: Arc, @@ -39,11 +39,11 @@ pub struct GrammarQueries { /// Finds the regions of a document written in other languages (tree-sitter /// injections), such as the scripts and styles of a Svelte component. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] pub struct Injector(Arc); -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Injector { /// The single-range regions `tree`, parsed from `source` with this /// injector's language, embeds in a named language, in match order. @@ -81,9 +81,9 @@ pub enum LanguageStatus { /// /// Grammars come from, in order: /// -/// 1. local pack roots the app opts into ([`StoreConfig::local_packs`]), -/// such as packs bundled with the app or built by `syntax-pack` in -/// development; +/// 1. with the `engine` feature, local pack roots the app opts into +/// ([`StoreConfig::local_packs`]), such as packs bundled with the app +/// or built by `syntax-pack` in development; /// 2. with the `bundled` feature, the grammars compiled into the binary /// ([`StoreConfig::bundled`]); /// 3. with the `download` feature, the cache of a signed index the app @@ -95,7 +95,7 @@ pub enum LanguageStatus { /// UI thread; [`crate::HighlightWorker`] makes them on its own thread. #[derive(Clone, Default)] pub struct GrammarStore { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] inner: Option>, } @@ -111,7 +111,7 @@ impl GrammarStore { Self::default() } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn new(config: StoreConfig) -> Self { Self { inner: Some(Arc::new(Inner { @@ -126,7 +126,7 @@ impl GrammarStore { } pub fn retry(&self, tag: &str) -> bool { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { let tag = tag.to_ascii_lowercase(); let mut state = lock(&inner.state); @@ -149,14 +149,14 @@ impl GrammarStore { } pub fn catalog(&self) -> Catalog { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { return inner.catalog(); } Catalog::default() } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn parser(&self, tag: &str) -> Option<(tree_sitter::Language, GrammarQueries)> { match self.grammar(&LanguageId::from_fence(tag)?) { Tag::Ready(grammar) => Some(( @@ -175,7 +175,7 @@ impl GrammarStore { /// The injection search of `tag`'s grammar, when it is loaded and has /// an injection query. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn injector(&self, tag: &str) -> Option { match self.grammar(&LanguageId::from_fence(tag)?) { Tag::Ready(grammar) if grammar.has_injections() => Some(Injector(grammar)), @@ -184,15 +184,15 @@ impl GrammarStore { } pub fn subscribe_status(&self, listener: impl Fn() -> bool + Send + 'static) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { lock(&inner.status_listeners).push(Box::new(listener)); } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = listener; } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn grammar(&self, language: &LanguageId) -> Tag { match &self.inner { Some(inner) => inner.lookup(language.as_str()), @@ -200,14 +200,14 @@ impl GrammarStore { } } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn ships_tags(&self, language: &LanguageId) -> bool { self.inner .as_ref() .is_some_and(|inner| inner.ships_tags(language.as_str())) } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub fn tree_sitter_language(&self, language: &LanguageId) -> Option { match self.grammar(language) { Tag::Ready(grammar) => Some(grammar.language().clone()), @@ -218,7 +218,7 @@ impl GrammarStore { /// Resolves `language`, loading its pack or starting its download on /// first use. pub fn status(&self, language: &LanguageId) -> LanguageStatus { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { return match inner.lookup(language.as_str()) { Tag::Ready(_) => LanguageStatus::Ready, @@ -234,11 +234,11 @@ impl GrammarStore { /// language resolves, whether its grammar arrived or failed. Return /// `false` to unsubscribe. Apps use it to wake their event loop. pub fn subscribe(&self, listener: impl Fn() -> bool + Send + 'static) { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] if let Some(inner) = &self.inner { lock(&inner.listeners).push(Box::new(listener)); } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] let _ = listener; } @@ -274,7 +274,7 @@ impl GrammarStore { source: &str, cancelled: &dyn Fn() -> bool, ) -> Outcome { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { self.highlight_within( language, @@ -283,7 +283,7 @@ impl GrammarStore { cancelled, ) } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (language, source, cancelled); Outcome::default() @@ -347,7 +347,7 @@ impl GrammarStore { focus: &dyn Fn() -> Option, emit: &mut dyn FnMut(Part), ) -> Outcome { - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] { self.highlight_windowed( language, @@ -359,7 +359,7 @@ impl GrammarStore { emit, ) } - #[cfg(not(feature = "engine"))] + #[cfg(not(feature = "runtime"))] { let _ = (language, source, slice, cancelled, focus, emit); Outcome::default() @@ -367,7 +367,7 @@ impl GrammarStore { } /// [`Self::highlight_streamed`] with explicit window sizes. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] #[allow(clippy::too_many_arguments)] pub(crate) fn highlight_windowed( &self, @@ -432,7 +432,7 @@ impl GrammarStore { } /// [`Self::highlight_until`] with explicit bounds on embedded layers. - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] pub(crate) fn highlight_within( &self, language: &LanguageId, @@ -484,7 +484,7 @@ pub(crate) struct Outcome { /// Bounds on embedded layers left some regions unparsed. #[cfg_attr(not(test), allow(dead_code))] pub(crate) truncated: bool, - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] #[cfg_attr(not(test), allow(dead_code))] pub(crate) work: crate::engine::Work, /// The spans went out window by window as [`Part`]s; `spans` is empty. @@ -497,7 +497,7 @@ pub(crate) struct Outcome { /// Where a streamed highlight starts its exact pass, and when it should /// stop to let other work run: asked after each window it finishes, with /// how many it finished in this call. -#[cfg_attr(not(feature = "engine"), allow(dead_code))] +#[cfg_attr(not(feature = "runtime"), allow(dead_code))] pub(crate) struct Slice<'a> { pub(crate) from: usize, /// The inexact window an earlier turn colored last. @@ -534,20 +534,22 @@ impl Outcome { /// What a [`GrammarStore`] may load. Nothing is loaded unless the app /// names it here: there is no default pack directory and no default index. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Debug, Clone, Default)] pub struct StoreConfig { + #[cfg(feature = "engine")] local_packs: Vec, #[cfg(feature = "bundled")] bundled: bool, /// Languages whose local packs stay pending until /// [`GrammarStore::release_deferred`], as if they were downloading. + #[cfg(feature = "engine")] deferred: Vec, #[cfg(feature = "download")] pub(crate) downloads: Option, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl StoreConfig { pub fn new() -> Self { Self::default() @@ -558,6 +560,7 @@ impl StoreConfig { /// loading one runs its code, so only name directories that only the /// app's own installer or the developer can write. Roots are searched /// in the order added, before any download. + #[cfg(feature = "engine")] pub fn local_packs(mut self, root: impl Into) -> Self { self.local_packs.push(root.into()); self @@ -567,6 +570,7 @@ impl StoreConfig { /// until [`GrammarStore::release_deferred`], the way a download leaves /// them, so tests can make a grammar arrive late without a server. #[doc(hidden)] + #[cfg(feature = "engine")] pub fn defer(mut self, languages: &[&str]) -> Self { self.deferred .extend(languages.iter().map(|l| (*l).to_owned())); @@ -589,7 +593,7 @@ impl StoreConfig { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] // Only downloads leave a language pending. #[cfg_attr(not(feature = "download"), allow(dead_code))] @@ -599,10 +603,10 @@ pub(crate) enum Tag { Unavailable, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] type Listener = Box bool + Send>; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct Inner { pub(crate) config: StoreConfig, pub(crate) state: Mutex, @@ -612,29 +616,33 @@ pub(crate) struct Inner { pub(crate) fetcher: Mutex>>, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Default)] pub(crate) struct State { /// Every tag asked for so far and what it resolved to. pub(crate) tags: HashMap, /// Tag to pack directory over the local roots, scanned on first lookup. + #[cfg(feature = "engine")] local: Option)>>, /// Grammars by where they came from, so a language's aliases share one /// load. `None` records a grammar that failed to load. pub(crate) loaded: HashMap>>, /// [`StoreConfig::defer`]red packs have been released. + #[cfg(feature = "engine")] released: bool, /// Tags left pending by [`StoreConfig::defer`]. + #[cfg(feature = "engine")] deferred: Vec, #[cfg(feature = "download")] pub(crate) remote: crate::download::Remote, } /// Where a grammar comes from. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(crate) enum Origin { /// A pack directory, local or in the download cache. + #[cfg(feature = "engine")] Pack(PathBuf), /// A grammar compiled in, by name. #[cfg_attr(not(feature = "bundled"), allow(dead_code))] @@ -642,7 +650,7 @@ pub(crate) enum Origin { } /// A language the catalog lists. -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] struct Known { origin: Origin, language: String, @@ -662,7 +670,7 @@ impl Known { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { // A panic while holding the lock leaves maps that are still valid. mutex @@ -670,7 +678,7 @@ pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Inner { pub(crate) fn lookup(self: &Arc, tag: &str) -> Tag { let mut state = lock(&self.state); @@ -687,14 +695,18 @@ impl Inner { } fn ships_tags(&self, tag: &str) -> bool { + // Only pack sources look past the tags already resolved. + #[cfg_attr(not(feature = "engine"), allow(unused_mut))] let mut state = lock(&self.state); if let Some(Tag::Ready(grammar)) = state.tags.get(tag) { return grammar.tags().is_some(); } - let local = state + #[cfg(feature = "engine")] + if let Some((_, manifest)) = state .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)); - if let Some((_, manifest)) = local.get(tag) { + .get_or_insert_with(|| scan_local(&self.config.local_packs)) + .get(tag) + { return manifest.tags.is_some(); } #[cfg(feature = "bundled")] @@ -711,10 +723,13 @@ impl Inner { } fn resolve(self: &Arc, state: &mut State, tag: &str) -> Tag { - let local = state + #[cfg(feature = "engine")] + if let Some((dir, manifest)) = state .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)); - if let Some((dir, manifest)) = local.get(tag).cloned() { + .get_or_insert_with(|| scan_local(&self.config.local_packs)) + .get(tag) + .cloned() + { if !state.released && self.config.deferred.contains(&manifest.language) { state.deferred.push(tag.to_owned()); return Tag::Pending; @@ -737,18 +752,21 @@ impl Inner { } fn catalog(&self) -> Catalog { + #[cfg_attr(not(feature = "engine"), allow(unused_mut))] let mut state = lock(&self.state); - let local = state - .local - .get_or_insert_with(|| scan_local(&self.config.local_packs)) - .clone(); let mut known: Vec = Vec::new(); - let mut seen = std::collections::HashSet::new(); - let mut local_packs: Vec<_> = local.values().cloned().collect(); - local_packs.sort_by(|a, b| a.0.cmp(&b.0)); - for (dir, manifest) in local_packs { - if seen.insert(manifest.language.clone()) { - known.push(Known::pack(dir, &manifest, true)); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + #[cfg(feature = "engine")] + { + let local = state + .local + .get_or_insert_with(|| scan_local(&self.config.local_packs)); + let mut local_packs: Vec<_> = local.values().cloned().collect(); + local_packs.sort_by(|a, b| a.0.cmp(&b.0)); + for (dir, manifest) in local_packs { + if seen.insert(manifest.language.clone()) { + known.push(Known::pack(dir, &manifest, true)); + } } } #[cfg(feature = "bundled")] @@ -815,6 +833,7 @@ impl Inner { catalog } + #[cfg(feature = "engine")] pub(crate) fn notify(&self) { lock(&self.listeners).retain(|listener| listener()); self.notify_status(); @@ -833,11 +852,11 @@ pub(crate) fn load(state: &mut State, dir: &Path, manifest: &PackManifest) -> Ta }) } -#[cfg(feature = "engine")] -pub(crate) fn load_with( +#[cfg(feature = "runtime")] +pub(crate) fn load_with( state: &mut State, origin: Origin, - load: impl FnOnce() -> Result, + load: impl FnOnce() -> Result, ) -> Tag { let grammar = state .loaded diff --git a/crates/diffy-syntax/src/symbols.rs b/crates/diffy-syntax/src/symbols.rs index 7b123e28..a438a673 100644 --- a/crates/diffy-syntax/src/symbols.rs +++ b/crates/diffy-syntax/src/symbols.rs @@ -1,7 +1,7 @@ -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use std::sync::Arc; -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] use crate::{GrammarStore, LanguageId}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -40,7 +40,7 @@ impl SymbolKind { .unwrap_or(Self::Other) } - #[cfg(feature = "engine")] + #[cfg(feature = "runtime")] fn from_capture(name: &str) -> Option { let kind = name.strip_prefix("definition.")?; Some(match kind { @@ -68,7 +68,7 @@ pub struct Symbol { pub end_line: u32, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub(crate) struct TagsQuery { query: tree_sitter::Query, kinds: Vec>, @@ -76,7 +76,7 @@ pub(crate) struct TagsQuery { fingerprint: u64, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl TagsQuery { pub(crate) fn compile( grammar: &tree_sitter::Language, @@ -148,7 +148,7 @@ impl TagsQuery { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] fn fnv1a(parts: &[&[u8]]) -> u64 { let mut hash: u64 = 0xcbf2_9ce4_8422_2325; for part in parts { @@ -162,14 +162,14 @@ fn fnv1a(parts: &[&[u8]]) -> u64 { hash } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] #[derive(Clone)] pub struct Tagger { grammar: Arc, query: Arc, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl std::fmt::Debug for Tagger { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Tagger") @@ -179,14 +179,14 @@ impl std::fmt::Debug for Tagger { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] pub enum TaggerLookup { Ready(Tagger), Pending, Unsupported, } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl Tagger { pub fn fingerprint(&self) -> u64 { self.query.fingerprint @@ -204,7 +204,7 @@ impl Tagger { } } -#[cfg(feature = "engine")] +#[cfg(feature = "runtime")] impl GrammarStore { pub fn tagger(&self, language: &LanguageId) -> TaggerLookup { if !self.ships_tags(language) { From 46f3969c441be955b72f550585bbadcd0c43e045 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:24:19 +0000 Subject: [PATCH 13/16] docs(node): trim the README --- bindings/node/README.md | 131 ++++++++++------------------------------ 1 file changed, 31 insertions(+), 100 deletions(-) diff --git a/bindings/node/README.md b/bindings/node/README.md index ffc45a29..407ebd9d 100644 --- a/bindings/node/README.md +++ b/bindings/node/README.md @@ -1,94 +1,45 @@ # @diffy/node -Diffy's file diff for Node. You read the files (from Git or anywhere else) -and pass in their contents. You get back a JSON document to render: a -semantic diff where a grammar exists, a line diff otherwise, with optional -syntax spans. The output format is specified in -[FORMAT.md](https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md), -and `index.d.ts` types every field. +Semantic file diffs for Node. Pass file contents, get back a JSON document +to render. The format is in +[FORMAT.md](https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md); +`index.d.ts` types it. ```js -const { diff, diffJson, expand, configure, languages, FORMAT_VERSION } = require('@diffy/node') +const { diff, expand } = require('@diffy/node') const doc = await diff( - { - files: [{ status: 'modified', old_path: 'src/a.rs', new_path: 'src/a.rs', old: oldBytes, new: newBytes }], - syntax: true, - budget_ms: 2000, - }, - { signal: request.signal }, + { files: [{ status: 'modified', old_path: 'a.rs', new_path: 'a.rs', old, new: next }], syntax: true, budget_ms: 2000 }, + { signal }, ) - -// Later, when the user expands a collapsed gap: -const rows = await expand(sameFileInput, doc.files[0].gap_after, { syntax: true }) +const rows = await expand(file, doc.files[0].gap_after, { syntax: true }) ``` -- `diff(request, { signal })` resolves to a `Document`, with files in the - order given. Contents are `Uint8Array`s (including `Buffer`s) or strings, - which are taken as UTF-8. - - `budget_ms` caps the whole request, counted from the call, so time - spent waiting in the queue (see below) counts too. Files that start - after it get line diffs, with `fallback.kind === "request_budget"`. - With `syntax`, files projected after it also come back without syntax - spans or scope headers. - - `file_budget_ms` (default 400) caps each file's semantic diff. - - `syntax` adds syntax spans. `syntax_max_bytes` (default 1 MiB) skips - them for larger sides. -- `diffJson(request, { signal })` is `diff`, but resolves to the document's - JSON text instead of parsing it. A server that only forwards the document - saves a `JSON.parse` and a `JSON.stringify` on its event loop (parsing a - 16 MB document blocks it for about 160 ms). -- `expand(file, gap, { syntax })` returns the context rows of a `Gap`, or of - any sub-range of one. `file` must have the contents the document was - made from. -- `configure({ max_concurrent_requests })` sets how many `diff` and - `diffJson` calls run at once, process-wide, and returns the settings; - `configure()` only returns them. See below. -- `languages()` lists the grammars compiled into this build. Every other - language gets a line diff. -- `FORMAT_VERSION` is the format version of the returned documents. +## API -Errors: +- `diff(request, { signal })` → `Document`. Contents are bytes or UTF-8 strings. + - `budget_ms`: whole request, from the call. Late files get line diffs (`request_budget`) and no syntax. + - `file_budget_ms`: per-file semantic budget, default 400. + - `syntax`, `syntax_max_bytes` (default 1 MiB): syntax spans. +- `diffJson(request, { signal })` → the same document as a JSON string, for forwarding without parsing. +- `expand(file, gap, { syntax })` → context rows of a gap or part of one. +- `configure({ max_concurrent_requests })`: concurrent `diff` calls, default cores / 4. Extra calls queue in order. +- `languages()`: grammars compiled in. Others get line diffs. +- `FORMAT_VERSION`. -- Aborting `signal` rejects at once with an `AbortError` whose `cause` is - the signal's reason. If the signal is already aborted, the call rejects - without starting any work. -- Malformed input rejects with a `TypeError` whose `code` is `"InvalidArg"`. - `configure` throws the same error. +An aborted `signal` rejects with `AbortError`. Bad input rejects with a +`TypeError` whose `code` is `"InvalidArg"`. -## How it runs +## Notes -- Each call copies its input on the JS thread. The work then runs on a - thread of its own, not on libuv's threadpool, so a long diff never holds - up the server's file system, DNS, or zlib calls. -- A diff of four or more files fans out to four worker threads, so `diff` - calls share one FIFO queue: at most `max_concurrent_requests` run at - once, by default the available cores divided by 4 (at least 1), which - keeps every core busy without time-slicing each other's file budgets. - Waiting holds no thread. A call aborted while it waits rejects at once - and never starts. Diffs of one to three files use one thread each, so a - server that sends files one at a time may want a higher limit. -- `expand` is not queued: it is one file on one thread, and someone is - waiting on it, so it should not wait behind whole diffs. -- The result crosses into JS as a single JSON string, which `diff` and - `expand` parse there. On an 11 MB document this cost about 75 ms on the - JS thread (29 ms to create the string, 45 ms to parse it). Building the - same objects through Node-API cost about 120 ms. `diffJson` skips the - parse. -- Grammars are compiled in. Nothing is downloaded and nothing is read from - disk. -- Renames of identifiers are detected but not vetted against the rest of - the repository, which the binding never sees. -- The binding uses only Node-API (version 4), so one binary loads in Node - 18 and later and in Electron. -- A Rust panic rejects the call instead of crashing the process. A crash - inside a grammar's C parser or scanner still takes the process down. +- Work runs on its own threads, never libuv's pool. `expand` skips the queue. +- No network or disk access: grammars are compiled in. +- Renames are detected but not checked against the rest of the repository. +- Node-API 4 only: one binary works in Node 18+ and Electron (unpack `*.node` from asar). +- A Rust panic rejects the call; a crash in a grammar's C code kills the process. ## Packages -`@diffy/node` has no dependencies of its own. It pulls in one prebuilt -binary as an optional dependency: - | Package | Platform | | --- | --- | | `@diffy/node-darwin-arm64` | macOS 11+, Apple silicon | @@ -97,37 +48,17 @@ binary as an optional dependency: | `@diffy/node-linux-arm64-gnu` | Linux arm64, glibc 2.28+ | | `@diffy/node-win32-x64-msvc` | Windows x64 | -There is no musl build. Installing with `--omit=optional` leaves the binary -out. Each platform package ships `THIRD_PARTY_NOTICES.md`, which lists the -grammars compiled in and their licenses, and `RUST_NOTICES.md`, which lists -the Rust crates compiled in with their licenses and license texts. - -For Electron, unpack `*.node` from the asar archive (for example with -electron-builder's `asarUnpack`). +Each ships `THIRD_PARTY_NOTICES.md` (grammars) and `RUST_NOTICES.md` (Rust crates). ## Development -From the repository root: - ```sh -# Grammars to compile in; rust and typescript are what the tests need. cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" rust typescript tsx cd bindings/node -DIFFY_SYNTAX_BUNDLE="$PWD/../../target/syntax-bundle" npm run build # writes diffy..node +DIFFY_SYNTAX_BUNDLE="$PWD/../../target/syntax-bundle" npm run build npm test ``` -The build uses the workspace's `node` profile: `release`, but with -`panic = "unwind"` so that panics can be caught. Set -`DIFFY_NODE_BINARY=path/to/diffy.node` to load a specific binary. - -`npm run notices -- --target TRIPLE --out DIR` writes `DIR/RUST_NOTICES.md` -for the crates `diffy-node` links on that target, and fails if any has a -license outside its allowlist of permissive ones. - -`.github/workflows/node.yml` builds and tests every platform. Linux builds -run inside manylinux_2_28, so their binaries need glibc 2.28 at most, and -CI checks this. Pushing a `node-v` tag publishes the platform -packages and then `@diffy/node`. Before tagging, set that version in -`package.json`, in every `npm/*/package.json`, and in the -`optionalDependencies`. +- `DIFFY_NODE_BINARY=path/to/diffy.node` loads a specific binary. +- `npm run notices -- --target TRIPLE --out DIR` writes `RUST_NOTICES.md`. +- Release: set the version in `package.json`, `npm/*/package.json` and `optionalDependencies`, then push a `node-v` tag. From e371b7506ab45578965ddc5e47e362b9b8a8a4b0 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:24:54 +0000 Subject: [PATCH 14/16] chore: name Rohit (seatedro) as copyright holder in the Apache licences syntax-pack keeps its own unmodified Apache-2.0 text for crediting nvim-treesitter's queries. --- bindings/node/LICENSE-APACHE | 2 +- crates/diffy-compare/LICENSE-APACHE | 2 +- crates/diffy-diff/LICENSE-APACHE | 2 +- crates/diffy-semantic/LICENSE-APACHE | 2 +- crates/diffy-syntax/LICENSE-APACHE | 2 +- tools/syntax-pack/APACHE-2.0.txt | 202 +++++++++++++++++++++++++++ tools/syntax-pack/src/main.rs | 2 +- 7 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 tools/syntax-pack/APACHE-2.0.txt diff --git a/bindings/node/LICENSE-APACHE b/bindings/node/LICENSE-APACHE index d6456956..9267b22b 100644 --- a/bindings/node/LICENSE-APACHE +++ b/bindings/node/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Rohit (seatedro) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crates/diffy-compare/LICENSE-APACHE b/crates/diffy-compare/LICENSE-APACHE index d6456956..9267b22b 100644 --- a/crates/diffy-compare/LICENSE-APACHE +++ b/crates/diffy-compare/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Rohit (seatedro) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crates/diffy-diff/LICENSE-APACHE b/crates/diffy-diff/LICENSE-APACHE index d6456956..9267b22b 100644 --- a/crates/diffy-diff/LICENSE-APACHE +++ b/crates/diffy-diff/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Rohit (seatedro) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crates/diffy-semantic/LICENSE-APACHE b/crates/diffy-semantic/LICENSE-APACHE index d6456956..9267b22b 100644 --- a/crates/diffy-semantic/LICENSE-APACHE +++ b/crates/diffy-semantic/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Rohit (seatedro) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crates/diffy-syntax/LICENSE-APACHE b/crates/diffy-syntax/LICENSE-APACHE index d6456956..9267b22b 100644 --- a/crates/diffy-syntax/LICENSE-APACHE +++ b/crates/diffy-syntax/LICENSE-APACHE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Rohit (seatedro) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/tools/syntax-pack/APACHE-2.0.txt b/tools/syntax-pack/APACHE-2.0.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/tools/syntax-pack/APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/syntax-pack/src/main.rs b/tools/syntax-pack/src/main.rs index c7e25748..64514895 100644 --- a/tools/syntax-pack/src/main.rs +++ b/tools/syntax-pack/src/main.rs @@ -1018,7 +1018,7 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result } /// The Apache License 2.0, verbatim. -const APACHE_2_0: &str = include_str!("../../../crates/diffy-syntax/LICENSE-APACHE"); +const APACHE_2_0: &str = include_str!("../APACHE-2.0.txt"); /// The repository, commit, and checkout of every source a language's /// build reads: its grammar, its query repository, and other entries' From bd1fc2e5639b6c10f89d55acb0ff5c65957f5755 Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:31:08 +0000 Subject: [PATCH 15/16] docs: cut comments and docs to what earns its place Drops narration, history, measurements and restated code across the new crates, the binding, the workflow and the docs; keeps invariants, coordinates, contracts and the reasons behind surprising choices. No code changes. --- .github/workflows/node.yml | 19 ++- AGENTS.md | 8 +- Cargo.toml | 3 +- README.md | 7 +- bindings/node/Cargo.toml | 16 +-- bindings/node/binding.js | 5 +- bindings/node/index.d.ts | 86 +++++-------- bindings/node/index.js | 8 +- bindings/node/npm/darwin-arm64/README.md | 2 +- bindings/node/npm/darwin-x64/README.md | 2 +- bindings/node/npm/linux-arm64-gnu/README.md | 2 +- bindings/node/npm/linux-x64-gnu/README.md | 2 +- bindings/node/npm/win32-x64-msvc/README.md | 2 +- bindings/node/scripts/build.mjs | 7 +- bindings/node/scripts/notices.mjs | 25 ++-- bindings/node/src/lib.rs | 41 +++---- bindings/node/src/queue.rs | 40 +++---- bindings/node/test/diffy.test.mjs | 5 +- crates/diffy-compare/Cargo.toml | 11 +- crates/diffy-compare/FORMAT.md | 126 +++++++++----------- crates/diffy-compare/src/document.rs | 4 +- crates/diffy-compare/src/file.rs | 8 +- crates/diffy-compare/src/lib.rs | 55 +++------ crates/diffy-compare/src/line.rs | 7 +- crates/diffy-compare/src/run.rs | 16 +-- crates/diffy-compare/src/wire.rs | 93 +++++---------- crates/diffy-compare/src/wire/tests.rs | 29 ++--- crates/diffy-syntax/Cargo.toml | 10 +- crates/diffy-syntax/PACKS.md | 36 ++---- crates/diffy-syntax/README.md | 17 ++- crates/diffy-syntax/build.rs | 18 ++- crates/diffy-syntax/src/bundled.rs | 18 ++- crates/diffy-syntax/src/engine.rs | 15 +-- crates/diffy-syntax/src/lib.rs | 14 +-- crates/diffy-syntax/src/store.rs | 6 +- crates/diffy-syntax/src/tests.rs | 18 +-- src/core/compare/semantic.rs | 3 +- src/core/error.rs | 2 +- tools/syntax-pack/README.md | 31 +++-- tools/syntax-pack/src/main.rs | 65 +++++----- 40 files changed, 345 insertions(+), 537 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 80e6b209..3c749eee 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -1,8 +1,5 @@ name: Node binding -# Builds @diffy/node for every platform it ships, tests each on a native -# runner, and publishes the packages on a `node-v` tag. - on: push: tags: ["node-v*"] @@ -50,8 +47,8 @@ jobs: target: x86_64-apple-darwin os: macos-26 node-arch: x64 - # Built and tested inside manylinux_2_28 (glibc 2.28), so the - # grammars' C and the Rust link both see that glibc and nothing newer. + # Built in manylinux_2_28 so the grammars' C and the Rust link see + # glibc 2.28 and nothing newer. - platform: linux-x64-gnu target: x86_64-unknown-linux-gnu os: ubuntu-24.04 @@ -73,10 +70,9 @@ jobs: PLATFORM: ${{ matrix.platform }} TARGET: ${{ matrix.target }} steps: - # Before checkout: it needs git for the submodule. The image's GCC - # toolset compiles the grammars and links Rust (in place of the clang - # .cargo/config.toml names, keeping its -fuse-ld=lld), so both build - # against the image's glibc. + # Before checkout, which needs git for the submodule. The image's GCC + # replaces .cargo/config.toml's clang (keeping -fuse-ld=lld) so the + # grammars and the Rust link use the image's glibc. - name: Install build tools (manylinux) if: matrix.container run: | @@ -154,8 +150,7 @@ jobs: cp "target/syntax-bundle/$TARGET/THIRD_PARTY_NOTICES.md" "$out/" ls -l "$out" - # Credits the Rust crates linked for this target, and fails on one - # whose license is not on the script's allowlist. + # Fails on a crate license outside the script's allowlist. - name: Write the Rust notices run: | node bindings/node/scripts/notices.mjs --locked --target "$TARGET" --out "bindings/node/npm/$PLATFORM" @@ -244,7 +239,7 @@ jobs: EOF # Platform packages first, so @diffy/node never names a missing one. - # A version already on npm is skipped, so a failed run can be rerun. + # Published versions are skipped, so a failed run can be rerun. - name: Publish env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index c4cab56a..3fe6b521 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,14 +27,14 @@ and syntax code stays here. - `src/apprt/` — runtime and services split per stream; all blocking Git/fs/network/AI work runs on workers and reports through `transport.rs`. - `src/core/` — product logic: compare backends, VCS, review, syntax, themes. -- `crates/diffy-compare/` — headless chain from a changed file's bytes to a - `DiffDocument` (semantic diff, else line diff); no UI, Git or network deps. +- `crates/diffy-compare/` — headless file bytes to `DiffDocument` (semantic, + else line diff); no UI, Git or network deps. - `crates/diffy-diff/` — diff model; `crates/diffy-ui/` — diff view on Quark; `crates/diffy-syntax/` — tree-sitter packs (see its `PACKS.md`); `tools/syntax-pack/` — pack builder; `registry.toml` is generated by `syntax-pack registry`, never edited by hand. -- `bindings/node/` — `@diffy/node`, the napi-rs binding over `diffy-compare` - with bundled grammars; build it with the `node` profile (see its README). +- `bindings/node/` — `@diffy/node`, napi-rs binding over `diffy-compare` with + bundled grammars; build with the `node` profile. - `crates/diffy-semantic/` — structural diff behind the semantic renderer; `tools/semantic-eval/` measures it against history and fixtures. - `crates/difftastic/` is vendored and out-of-workspace, built only for diff --git a/Cargo.toml b/Cargo.toml index 2ae622df..758f4521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,8 +138,7 @@ codegen-units = 1 panic = "abort" strip = true -# The Node binding (bindings/node): release, but a Rust panic unwinds to the -# binding boundary and rejects the call instead of aborting the host process. +# bindings/node: a panic rejects the call instead of aborting the Node process. [profile.node] inherits = "release" panic = "unwind" diff --git a/README.md b/README.md index 652aaa42..3299bbc1 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ dx serve --hot-patch --features hot-reload ## License -The Diffy app is GPL-3.0. The headless crates the compare chain is built -from (`diffy-diff`, `diffy-semantic`, `diffy-syntax`, `diffy-compare`) and -the Node binding are MIT OR Apache-2.0; see the `LICENSE-MIT` and -`LICENSE-APACHE` files in each. +- The app: GPL-3.0. +- `diffy-diff`, `diffy-semantic`, `diffy-syntax`, `diffy-compare`, and the + Node binding: MIT OR Apache-2.0 (`LICENSE-MIT`, `LICENSE-APACHE` in each). diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index c91aaec6..5429ffff 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -10,23 +10,19 @@ publish = false [lib] crate-type = ["cdylib"] -# Build with `--profile node` (see the workspace manifest): release panics -# abort, which would take the host Node process down with them. +# Build with `--profile node`: release's panic = "abort" would kill the host. [dependencies] diffy-compare = { path = "../../crates/diffy-compare", features = ["serde"] } # Grammars compiled in from $DIFFY_SYNTAX_BUNDLE; no packs, no network. diffy-syntax = { path = "../../crates/diffy-syntax", features = ["bundled"] } -# Node-API bindings. `napi4` is the lowest version with thread-safe -# functions, which settle a promise from the request's own thread; -# `dyn-symbols` resolves Node-API from the host process at load time, so -# the same file loads in Node and Electron and needs no node.lib on Windows. +# `napi4`: lowest with thread-safe functions. `dyn-symbols`: Node-API is +# resolved from the host at load, so one file loads in Node and Electron and +# Windows needs no node.lib. napi = { version = "3", default-features = false, features = ["napi4", "dyn-symbols"] } -# The `#[napi]` attribute. Without `type-def`: index.d.ts is written by -# hand from FORMAT.md, which generated types cannot express. +# No `type-def`: index.d.ts is hand-written from FORMAT.md. napi-derive = { version = "3", default-features = false, features = ["strict"] } -# Results cross into JS as one JSON string. serde_json = "1.0" [build-dependencies] -# Emits the platform link flags a Node addon needs (macOS dynamic lookup). +# Addon link flags (macOS dynamic lookup). napi-build = "2" diff --git a/bindings/node/binding.js b/bindings/node/binding.js index 1cc504fc..3e382f7d 100644 --- a/bindings/node/binding.js +++ b/bindings/node/binding.js @@ -1,8 +1,7 @@ 'use strict' -// Loads the native addon: $DIFFY_NODE_BINARY (a path, from the working -// directory) if set, else a local build beside this file (`npm run build`), -// else this platform's package. +// Loads $DIFFY_NODE_BINARY (resolved from the cwd), else a local +// `npm run build` output, else this platform's package. const { existsSync, readFileSync } = require('node:fs') const { join, resolve } = require('node:path') diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index 9151daaf..ecdc33a1 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -1,55 +1,39 @@ -// Types for @diffy/node. The output types follow Diffy's display format, -// version 1; the spec of every field is crates/diffy-compare/FORMAT.md: +// Output follows Diffy's display format, version 1, specified in // https://github.com/seatedro/diffy/blob/master/crates/diffy-compare/FORMAT.md -// -// Clients must ignore fields they do not know: adding an optional field -// does not change FORMAT_VERSION. +// Ignore unknown fields: new optional fields keep FORMAT_VERSION. -/** The display format version of every Document this build returns. */ +/** Display format version of every Document this build returns. */ export const FORMAT_VERSION: number -/** Canonical names of the grammars compiled into this build, e.g. `"rust"`. */ +/** Canonical names of the bundled grammars, e.g. `"rust"`. */ export function languages(): string[] /** - * Diffs every file off the JS thread. Resolves with one File per input, - * in input order. Waits in a process-wide FIFO queue while - * `max_concurrent_requests` others run (see `configure`). - * - * Rejects with an `AbortError` (`name === "AbortError"`, `code === - * "ABORT_ERR"`, `cause` the signal's reason) when `options.signal` aborts, - * and with a `TypeError` whose `code` is `"InvalidArg"` for malformed input. + * Diffs off the JS thread; one File per input, in order. Waits in a + * process-wide FIFO while `max_concurrent_requests` others run. + * Rejects with an `AbortError` (`code: "ABORT_ERR"`, `cause`: the signal's + * reason) on abort, and a `TypeError` (`code: "InvalidArg"`) for bad input. */ export function diff(request: DiffRequest, options?: DiffOptions): Promise -/** - * `diff`, resolving to the Document as the JSON text it was built as, not - * parsed: for a server forwarding it to clients without parsing it on its - * event loop. `JSON.parse` of it equals what `diff` resolves to. - */ +/** `diff`, unparsed: forward the Document's JSON without parsing it on the event loop. */ export function diffJson(request: DiffRequest, options?: DiffOptions): Promise /** - * The context rows of `gap` (a Gap from a Document, or any sub-range of - * one) of `file`, which must hold the same contents the Document was made - * from. Fewer rows than `gap.len` come back when the range runs past the - * end of either side or the sides no longer match there. Rejects with a - * `TypeError` whose `code` is `"InvalidArg"` for malformed input. + * Context rows of `gap` (a Document's Gap or a sub-range of one); `file` must + * hold the contents the Document was made from. Fewer than `gap.len` rows past + * either side's end or where the sides differ. Bad input rejects as in `diff`. */ export function expand(file: FileInput, gap: Gap, options?: ExpandOptions): Promise -/** - * Sets the given options, process-wide, and returns all of them. Throws a - * `TypeError` whose `code` is `"InvalidArg"` for an invalid value. - */ +/** Sets the given options process-wide; returns all. Throws a `TypeError` (`code: "InvalidArg"`) for a bad value. */ export function configure(options?: Configuration): Required export interface Configuration { /** - * The most `diff` and `diffJson` calls that run at once; later ones wait - * their turn. A positive integer. Default: the available cores divided - * by 4, at least 1. Raising it starts waiting requests at once; lowering - * it lets running ones finish. `expand` is never queued. + * Concurrent `diff`/`diffJson` calls; the rest queue. Positive integer. + * Default: cores / 4, at least 1. Raising it starts queued calls at once; + * lowering it lets running ones finish. `expand` is never queued. */ max_concurrent_requests?: number } @@ -59,9 +43,8 @@ export interface Configuration { export type Status = 'modified' | 'added' | 'deleted' | 'renamed' | 'copied' /** - * One changed file as the caller read it. Contents are bytes; a string is - * taken as its UTF-8 encoding. Missing contents are empty. At least one - * path is required: the language comes from `new_path ?? old_path`. + * Contents are bytes; a string means its UTF-8. Missing contents are empty. + * Needs a path; the language comes from `new_path ?? old_path`. */ export interface FileInput { status: Status @@ -76,14 +59,12 @@ export interface FileInput { export interface DiffRequest { files: FileInput[] /** - * Time budget for the whole request, from the call, so it includes time - * queued. Files started after it get line diffs, with - * `fallback.kind === "request_budget"`. - * With `syntax`, files projected after it get no syntax spans or scope - * headers. + * Whole-request budget from the call, time queued included. Files started + * after it get line diffs (`fallback.kind: "request_budget"`); with + * `syntax`, files projected after it get no syntax spans or scope headers. */ budget_ms?: number - /** Time budget for each file's semantic diff. Default 400. */ + /** Per-file semantic diff budget. Default 400. */ file_budget_ms?: number /** Add syntax spans to rows. Default false. */ syntax?: boolean @@ -108,7 +89,7 @@ export interface ExpandOptions { export interface Document { version: 1 - /** In the order the files were requested. */ + /** In request order. */ files: File[] } @@ -119,9 +100,8 @@ export interface File { /** Absent for a deleted file. Show `new_path ?? old_path`. */ new_path?: string /** - * The language tag the path names: the extension (`rs`, `ts`) or, for a - * file without one, its lowercased name (`dockerfile`). Absent when the - * path names none, or when `fallback.kind` is `no_grammar`. + * The extension (`rs`) or, without one, the lowercased name (`dockerfile`). + * Absent when the path names none or `fallback.kind` is `no_grammar`. */ language?: string /** A binary file has no `hunks` and no `gap_after`. */ @@ -136,10 +116,7 @@ export interface File { fallback?: Fallback /** In file order. Render each hunk's `gap_before`, header, rows; then `gap_after`. */ hunks?: Hunk[] - /** - * Unchanged lines after the last hunk. For a file with no hunks - * (unchanged, or renamed without edits) this is the whole file. - */ + /** Unchanged lines after the last hunk; the whole file when there are no hunks. */ gap_after?: Gap /** Items the semantic diff found changed: new-side items, then old-side ones. */ outline?: OutlineItem[] @@ -151,10 +128,9 @@ export interface File { export interface Fallback { /** - * `no_grammar`: no language is known for the path. `grammar_unavailable`: - * the path names a language whose grammar could not load. `semantic`: the - * semantic diff gave up, see `reason`. `request_budget`: the request's - * deadline passed before this file was diffed. + * `no_grammar`: no language for the path. `grammar_unavailable`: its grammar + * failed to load. `semantic`: the semantic diff gave up, see `reason`. + * `request_budget`: the request deadline passed first. */ kind: 'no_grammar' | 'grammar_unavailable' | 'semantic' | 'request_budget' /** Present when `kind` is `semantic`. */ @@ -169,9 +145,9 @@ export interface Hunk { old_len: number new_start: number new_len: number - /** The item enclosing the hunk's first change, for its header line. */ + /** Item enclosing the first change, for the header line. */ header?: string - /** Unchanged lines between the previous hunk (or the top of the file) and this one. */ + /** Unchanged lines since the previous hunk or the top of the file. */ gap_before?: Gap /** In display order; each side's line numbers ascend. */ rows: Row[] diff --git a/bindings/node/index.js b/bindings/node/index.js index 4aaa5270..384d943f 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -24,8 +24,8 @@ async function diffJson(request, options) { const work = call(() => native.diff(request, token)) if (signal === undefined) return work - // Abort rejects at once; a queued native request never starts, a - // running one stops at its next file, and its own rejection is dropped. + // Abort rejects at once and cancels the native request; its own + // rejection is dropped. let onAbort const aborted = new Promise((_, reject) => { onAbort = () => { @@ -57,8 +57,8 @@ function configure(options) { return { max_concurrent_requests: state.limit } } -// The binding throws synchronously only for its arguments (each conversion -// with its own code), or with GenericFailure when it cannot start a thread. +// Sync throws are argument errors (codes vary by conversion), except +// GenericFailure: no thread could start. function call(binding) { try { return binding() diff --git a/bindings/node/npm/darwin-arm64/README.md b/bindings/node/npm/darwin-arm64/README.md index 7d66cffd..f3e182c3 100644 --- a/bindings/node/npm/darwin-arm64/README.md +++ b/bindings/node/npm/darwin-arm64/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Apple silicon. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/darwin-x64/README.md b/bindings/node/npm/darwin-x64/README.md index 41970f42..e16a890c 100644 --- a/bindings/node/npm/darwin-x64/README.md +++ b/bindings/node/npm/darwin-x64/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for macOS on Intel. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/linux-arm64-gnu/README.md b/bindings/node/npm/linux-arm64-gnu/README.md index 389e4920..7a4374f8 100644 --- a/bindings/node/npm/linux-arm64-gnu/README.md +++ b/bindings/node/npm/linux-arm64-gnu/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux arm64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/linux-x64-gnu/README.md b/bindings/node/npm/linux-x64-gnu/README.md index 69145542..ce6d7c0e 100644 --- a/bindings/node/npm/linux-x64-gnu/README.md +++ b/bindings/node/npm/linux-x64-gnu/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Linux x64 with glibc 2.28 or newer. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/npm/win32-x64-msvc/README.md b/bindings/node/npm/win32-x64-msvc/README.md index 74b36a0c..8adaeb79 100644 --- a/bindings/node/npm/win32-x64-msvc/README.md +++ b/bindings/node/npm/win32-x64-msvc/README.md @@ -2,4 +2,4 @@ The prebuilt [`@diffy/node`](https://www.npmjs.com/package/@diffy/node) binary for Windows x64. Install `@diffy/node`, not this package. -`THIRD_PARTY_NOTICES.md` lists the tree-sitter grammars compiled into it and their licenses. `RUST_NOTICES.md` lists the Rust crates compiled into it, with their licenses. +`THIRD_PARTY_NOTICES.md` credits the tree-sitter grammars compiled in; `RUST_NOTICES.md` the Rust crates. diff --git a/bindings/node/scripts/build.mjs b/bindings/node/scripts/build.mjs index 2c0b964f..0ddc2043 100644 --- a/bindings/node/scripts/build.mjs +++ b/bindings/node/scripts/build.mjs @@ -1,10 +1,9 @@ -// Builds the addon with the `node` profile and copies it to -// `diffy..node`, the name binding.js and the platform packages use. +// Builds the addon as `diffy..node`, the name binding.js and the +// platform packages load. // // node scripts/build.mjs [--target TRIPLE] [--out DIR] [--profile NAME] [--locked] // -// Set DIFFY_SYNTAX_BUNDLE to the absolute `syntax-pack bundle --out` directory -// to compile grammars in; without it the build has none. +// DIFFY_SYNTAX_BUNDLE: absolute `syntax-pack bundle --out` dir; unset, no grammars. import { spawnSync } from 'node:child_process' import { copyFileSync, mkdirSync } from 'node:fs' diff --git a/bindings/node/scripts/notices.mjs b/bindings/node/scripts/notices.mjs index 224962be..51d8f4c0 100644 --- a/bindings/node/scripts/notices.mjs +++ b/bindings/node/scripts/notices.mjs @@ -1,11 +1,8 @@ -// Writes RUST_NOTICES.md: every Rust crate compiled into the addon for a -// target, with its license and the license files it ships. Fails, writing -// nothing, if a crate's license is not on the permissive allowlist. +// Writes RUST_NOTICES.md: each Rust crate linked into the addon for a target, +// with its license files. Fails, writing nothing, on a license outside the +// allowlist. Grammars are credited in the bundle's THIRD_PARTY_NOTICES.md. // // node scripts/notices.mjs [--target TRIPLE] [--out DIR] [--locked] -// -// The grammars compiled in are credited separately, in the bundle's -// THIRD_PARTY_NOTICES.md. import { spawnSync } from 'node:child_process' import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' @@ -14,8 +11,8 @@ import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' const PACKAGE = 'diffy-node' -// SPDX identifiers that need no more than this notice. MPL-2.0 is -// file-level copyleft: it binds only the crate's own files, unmodified here. +// Licenses this notice satisfies. MPL-2.0 binds only the crate's own files, +// shipped unmodified. const PERMITTED = new Set([ 'MIT', 'Apache-2.0', @@ -31,8 +28,7 @@ const PERMITTED = new Set([ ]) const PERMITTED_EXCEPTIONS = new Set(['LLVM-exception']) const LICENSE_FILE = /^(licen[cs]e|copying|notice)/i -// Crates whose packages ship no license file, with the text from their -// repository, so their copyright notice is still reproduced. +// napi-rs crates ship no license file; reproduce their repository's. const NAPI_RS = { path: resolve(dirname(fileURLToPath(import.meta.url)), '../notices/napi-rs-LICENSE'), source: 'github.com/napi-rs/napi-rs' } const FALLBACK_LICENSES = new Map(['napi', 'napi-sys', 'napi-derive', 'napi-derive-backend'].map((name) => [name, NAPI_RS])) @@ -47,10 +43,9 @@ const { values } = parseArgs({ const target = values.target ?? hostTriple() const locked = values.locked ? ['--locked'] : [] -// The crate set comes from `cargo tree`, which resolves features for this -// package alone, as `cargo build -p` does. `cargo metadata` resolves them -// for the whole workspace, so its graph also holds crates only the app -// links (its HTTP client and TLS stack). +// `cargo tree -p` resolves features for this package alone, as `cargo build +// -p` does; `cargo metadata` resolves the whole workspace and would add +// app-only crates (HTTP, TLS). const tree = cargo(['tree', '-p', PACKAGE, '-e', 'normal', '--target', target, '--prefix', 'none', '--format', '{p}', ...locked]) const used = new Set() for (const line of tree.split('\n')) { @@ -90,7 +85,7 @@ const out = [ '', `Diffy's own crates (${ours.map((pkg) => `\`${pkg.name}\``).join(', ')}) are ${[...new Set(ours.map((pkg) => pkg.license))].join('; ')}: see LICENSE-MIT and LICENSE-APACHE.`, ] -// Many crates ship the same license text; it is printed once. +// Identical license texts are printed once. const printed = new Map() for (const pkg of theirs) { out.push('', `## ${pkg.name} ${pkg.version}`, '', `License: ${pkg.license}`) diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index 81f0e5f0..0e9e846d 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -1,11 +1,9 @@ -//! The native half of `@diffy/node`; `index.js` wraps it and is the API. +//! Native half of `@diffy/node`; `index.js` is the API. //! -//! Each call copies its input on the JS thread, then does the work on a -//! thread of its own, never on libuv's pool (which a server's file system, -//! DNS, and zlib calls share), and settles its promise through a -//! thread-safe function. `diff` requests wait their turn in [`queue`]; -//! `expand` starts at once. Results cross as one JSON string of the -//! display format. +//! Input is copied on the JS thread; work runs on threads of its own, not +//! libuv's pool (shared with fs, DNS and zlib), and settles through a +//! thread-safe function. `diff` waits in [`queue`]; `expand` starts at once. +//! Results cross as one JSON string. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; @@ -23,7 +21,7 @@ mod queue; #[napi] pub const FORMAT_VERSION: u32 = wire::FORMAT_VERSION; -/// The grammars compiled into this build, by canonical name. +/// Bundled grammars, by canonical name. #[napi] pub fn languages() -> Vec { diffy_syntax::bundled::languages() @@ -31,8 +29,7 @@ pub fn languages() -> Vec { .collect() } -/// Built on first use, off the JS thread; bundled grammars only, so it -/// never touches the network or the file system. +/// Built lazily off the JS thread; bundled only, so no network or file system. fn grammars() -> &'static GrammarStore { static STORE: OnceLock = OnceLock::new(); STORE.get_or_init(|| GrammarStore::new(StoreConfig::new().bundled())) @@ -70,8 +67,7 @@ pub struct Gap { pub len: f64, } -/// Stops the request it is passed to; `index.js` sets it from an -/// `AbortSignal`. +/// Set by `index.js` from an `AbortSignal`. #[napi] pub struct CancelToken { flag: Arc, @@ -87,8 +83,8 @@ impl CancelToken { } } - /// Stops the request: a queued one is rejected at once and never - /// starts; a running one stops at its next file. + /// A queued request is rejected at once and never starts; a running one + /// stops at its next file. #[napi] pub fn cancel(&self) { self.flag.store(true, Ordering::Relaxed); @@ -98,14 +94,12 @@ impl CancelToken { #[napi(object, object_from_js = false)] pub struct QueueState { - /// The most `diff` requests that run at once. pub limit: u32, pub running: u32, pub queued: u32, } -/// Sets the most `diff` requests that run at once, if `limit` is given, -/// and reports the queue. +/// Sets the `diff` concurrency limit, if given, and reports the queue. #[napi(js_name = "queue")] pub fn queue_state(limit: Option) -> Result { if let Some(limit) = limit { @@ -114,7 +108,7 @@ pub fn queue_state(limit: Option) -> Result { "max_concurrent_requests: {limit} is not a positive integer" ))); } - // Saturates past usize::MAX, which means "no limit" anyway. + // `as` saturates; usize::MAX means no limit anyway. queue::set_limit(limit as usize); } let state = queue::state(); @@ -126,8 +120,7 @@ pub fn queue_state(limit: Option) -> Result { }) } -/// Diffs `request.files` and resolves to the display format's JSON, once -/// the request's turn in the queue comes. +/// Queued; resolves to the Document's JSON. #[napi] pub fn diff<'env>( env: &'env Env, @@ -158,8 +151,7 @@ pub fn diff<'env>( options.file_budget = file_budget; options.deadline = deadline; options.cancel = Some(&flag); - // No `vet`: renames are detected but not confirmed against the rest - // of the repository, which the binding never sees. + // No `vet`: the binding never sees the repository to confirm renames. let diffs = diffy_compare::diff_files(&inputs, &options).map_err(compare_error)?; let document = wire::document( &inputs, @@ -180,8 +172,7 @@ pub fn diff<'env>( Ok(promise) } -/// The context rows of `gap` in `file`, as the display format's JSON. -/// Never queued: it is one file on one thread, and a user waits on it. +/// Context rows of `gap` as JSON. Not queued: one file, and a user is waiting. #[napi] pub fn expand<'env>( env: &'env Env, @@ -229,7 +220,7 @@ impl WireSettings { fn wire_settings(syntax: Option, max_bytes: Option) -> Result { let syntax_max_bytes = match max_bytes { Some(bytes) if bytes.is_finite() && bytes >= 0.0 && bytes.fract() == 0.0 => { - // Saturates past usize::MAX, which means "no limit" anyway. + // `as` saturates; usize::MAX means no limit anyway. bytes as usize } Some(bytes) => return Err(invalid(format!("syntax_max_bytes: {bytes} is not a size"))), diff --git a/bindings/node/src/queue.rs b/bindings/node/src/queue.rs index 66b121c0..45c5d937 100644 --- a/bindings/node/src/queue.rs +++ b/bindings/node/src/queue.rs @@ -1,10 +1,8 @@ -//! The process-wide FIFO queue of `diff` requests. +//! Process-wide FIFO of `diff` requests. //! -//! At most [`limit`](Queue::limit) requests run at once, each on a thread -//! of its own that fans out to a few more. A queued request holds no thread -//! and blocks nothing: when a request finishes, its thread takes the next -//! queued one, or exits. Requests are cancelled out of the queue at once -//! and never start. +//! At most [`limit`](Queue::limit) run at once, each on its own thread. A +//! queued request holds no thread: a finishing thread takes the next one or +//! exits. A cancelled queued request is rejected at once and never starts. use std::collections::VecDeque; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -13,19 +11,16 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; use napi::{Env, Error, JsDeferred, Result, Status}; -/// The semantic diff may run on the request thread itself; give it the -/// stack the app gives its semantic workers. Only reserved, not committed. +/// The semantic diff may run on this thread; same stack as the app's +/// semantic workers. Reserved, not committed. pub(crate) const STACK_BYTES: usize = 16 << 20; -/// `diffy_compare` diffs a request of four or more files on four workers, -/// then highlights them on as many, so a request can keep this many cores -/// busy. +/// `diffy_compare`'s per-request worker fan-out. const WORKERS_PER_REQUEST: usize = 4; /// A promise settled with a JSON string from any thread. pub(crate) type Deferred = JsDeferred Result + Send>>; -/// A request's work: the JSON it resolves to. pub(crate) type Work = Box Result + Send>; pub(crate) struct Job { @@ -78,15 +73,13 @@ impl Queue { } } -/// One request per [`WORKERS_PER_REQUEST`] cores, so running requests keep -/// every core busy without time-slicing each other's file budgets away. +/// Fills the cores without requests time-slicing away each other's file +/// budgets. fn default_limit() -> usize { let cores = std::thread::available_parallelism().map_or(1, usize::from); (cores / WORKERS_PER_REQUEST).max(1) } -/// Queues `job`; it starts once it is the oldest and fewer than the limit -/// are running. pub(crate) fn submit(job: Job) { let mut queue = lock(); queue.waiting.push_back(job); @@ -97,8 +90,7 @@ pub(crate) fn submit(job: Job) { } } -/// Sets the limit and starts the queued jobs it now lets run. Running jobs -/// past a lowered limit finish; no new ones start until under it. +/// Raising the limit starts queued jobs; lowering it lets running ones finish. pub(crate) fn set_limit(limit: usize) { let mut queue = lock(); queue.limit = limit.max(1); @@ -127,8 +119,7 @@ pub(crate) fn state() -> State { } } -/// Rejects the queued jobs `cancel` stops, which then never start. Running -/// ones see the flag themselves. +/// Rejects queued jobs with this flag; running ones poll it themselves. pub(crate) fn cancel(cancel: &Arc) { let mut queue = lock(); let waiting = std::mem::take(&mut queue.waiting); @@ -153,11 +144,11 @@ fn finish() -> Option { queue.next() } -/// Runs `job`, which holds a slot, and then queued jobs, on a new thread. +/// Runs `job`, which already holds a slot, then queued jobs, on a new thread. fn start(mut job: Job) { loop { - // The job is sent once the thread exists: a failed spawn drops its - // closure, and the job's promise must still be settled. + // Sent only once the thread exists: a failed spawn drops its closure, + // and the promise must still be settled. let error = match spawn_runner() { Ok(sender) => match sender.send(job) { Ok(()) => return, @@ -193,8 +184,7 @@ fn spawn_runner() -> std::io::Result> { Ok(sender) } -/// Runs `work` and settles `deferred` with its result. A panic in `work` -/// rejects the promise; it cannot unwind into Node. +/// A panic in `work` rejects the promise instead of unwinding into Node. pub(crate) fn settle(deferred: Deferred, work: Work) { match catch_unwind(AssertUnwindSafe(work)) { Ok(Ok(json)) => deferred.resolve(Box::new(move |_| Ok(json))), diff --git a/bindings/node/test/diffy.test.mjs b/bindings/node/test/diffy.test.mjs index eb8dcb5b..fc6b8ae4 100644 --- a/bindings/node/test/diffy.test.mjs +++ b/bindings/node/test/diffy.test.mjs @@ -103,7 +103,6 @@ describe('abort', () => { await assert.rejects(running, { name: 'AbortError' }) }) - // Needs the compare core: without it the request fails before the abort. it('leaves the binding usable', async () => { const controller = new AbortController() const running = diff(bigRequest(256), { signal: controller.signal }) @@ -148,7 +147,6 @@ describe('invalid input rejects', () => { } }) -// These need the compare core (diffy_compare::diff_files). describe('diff', () => { it('diffs a modified file semantically', async () => { const doc = await diff({ files: [modified('src/view.rs', OLD_RS, NEW_RS)] }) @@ -269,7 +267,7 @@ describe('diffJson', () => { }) }) -// The queue is process-wide; every test puts the default limit back. +// The queue is process-wide; each test restores the default limit. describe('queue', () => { const { max_concurrent_requests: initial } = configure() const small = () => diff({ files: [modified('a.rs', 'a\n', 'b\n')] }) @@ -287,7 +285,6 @@ describe('queue', () => { for (let i = 0; i < 4; i++) done.push(small().then(() => order.push(i))) assert.deepEqual(native.queue(), { limit: 1, running: 1, queued: 4 }) await Promise.all(done) - // Unqueued, the small requests would finish long before the big one. assert.deepEqual(order, ['big', 0, 1, 2, 3]) assert.deepEqual(native.queue(), { limit: 1, running: 0, queued: 0 }) }) diff --git a/crates/diffy-compare/Cargo.toml b/crates/diffy-compare/Cargo.toml index c97619f6..6dd1c5dd 100644 --- a/crates/diffy-compare/Cargo.toml +++ b/crates/diffy-compare/Cargo.toml @@ -7,26 +7,21 @@ rust-version = "1.92" license = "MIT OR Apache-2.0" publish = false -# No UI, Git, or network dependencies: callers read the files and hand over -# their bytes. Grammars come from a diffy-syntax store the caller builds. [dependencies] diffy-diff = { path = "../diffy-diff" } diffy-semantic = { path = "../diffy-semantic" } diffy-syntax = { path = "../diffy-syntax", features = ["runtime"] } -# The line diff gix runs (histogram plus Git's slider heuristics), the same -# crate and version gix already pulls into the app, so line diffs match Git -# and the app links one copy. +# gix's own line diff: matches Git, and the app links one copy. imara-diff = { package = "gix-imara-diff", version = "0.2.1", default-features = false } tracing = "0.1.44" -# Only for `wire`. serde = { version = "1.0", default-features = false, features = ["derive", "std"], optional = true } [features] default = [] -# The versioned, serializable display format in `wire`. +# Enables `wire`. serde = ["dep:serde"] [dev-dependencies] serde_json = "1.0" -# Tests load grammar packs through `diffy_syntax::testing`. +# For `diffy_syntax::testing` packs. diffy-syntax = { path = "../diffy-syntax", features = ["engine"] } diff --git a/crates/diffy-compare/FORMAT.md b/crates/diffy-compare/FORMAT.md index e2bf0e73..53b3f108 100644 --- a/crates/diffy-compare/FORMAT.md +++ b/crates/diffy-compare/FORMAT.md @@ -1,40 +1,35 @@ # Diffy display format, version 1 -The JSON that Diffy's compare produces for clients to render diffs from -(`diffy_compare::wire`, behind the crate's `serde` feature). It is -layout-neutral: one row list renders both split and unified views, and -nothing needs recomputing on the client. +JSON produced by `diffy_compare::wire` (crate feature `serde`). Layout-neutral: +one row list renders both split and unified views. ## Versioning -Every document carries `version`. It changes only for a breaking change: -removing or renaming a field or enum value, changing a field's type, -meaning, or coordinates, or making an optional field required. Adding an -optional field is not breaking and keeps the version, so clients must -ignore fields they do not know. The one enum that may gain values without -a bump is the syntax `class`; treat unknown classes as plain text. +- `version` changes only on a breaking change: removing or renaming a field or + enum value, changing a field's type, meaning or coordinates, or making an + optional field required. +- Adding an optional field keeps the version, so ignore unknown fields. +- Syntax `class` may gain values without a bump; render unknown ones as plain. ## Conventions -- Field names are `snake_case`; enum values are `snake_case` strings. -- Optional fields are omitted when empty: an absent string or object means - none, an absent array means `[]`, an absent boolean means `false`. +- Field names and enum values are `snake_case`. +- Optional fields are omitted when empty: absent string or object means none, + absent array means `[]`, absent boolean means `false`. - Line numbers are 1-based. -- Ranges are half-open: `start` is included, `end` is not. -- **Span offsets are UTF-16 code units within the row's `text`**, the way - JavaScript indexes strings: `text.slice(span.start, span.end)` is the - span's text. -- `text` is one line as Diffy displays it: no line terminator, no trailing - `\r` of a CRLF line, and each byte that is not valid UTF-8 shown as - U+FFFD (`�`). Whether the last line ends with a newline is stated on the - file (`old_missing_newline`, `new_missing_newline`). +- Ranges are half-open: `start` included, `end` excluded. +- **Span offsets are UTF-16 code units within the row's `text`**: + `text.slice(span.start, span.end)` is the span's text. +- `text` is one display line: no line terminator, no trailing `\r`, invalid + UTF-8 bytes as U+FFFD (`�`). The final newline is reported per file + (`old_missing_newline`, `new_missing_newline`). ## Document | Field | Type | | | --- | --- | --- | | `version` | number | The format version, `1`. | -| `files` | File[] | In the order the files were requested. | +| `files` | File[] | In request order. | ## File @@ -43,7 +38,7 @@ a bump is the syntax `class`; treat unknown classes as plain text. | `status` | `"modified"` \| `"added"` \| `"deleted"` \| `"renamed"` \| `"copied"` | | | `old_path` | string? | Absent for an added file. | | `new_path` | string? | Absent for a deleted file. Show `new_path ?? old_path`. | -| `language` | string? | The language tag the path names, as Diffy names grammars: the extension (`rs`, `ts`, `py`) or, for a file without one, its lowercased name (`dockerfile`). Absent when the path names none, or when `fallback.kind` is `no_grammar`. | +| `language` | string? | The path's language tag, as Diffy names grammars: the extension (`rs`, `ts`, `py`), or the lowercased name of a file without one (`dockerfile`). Absent when the path names none, or when `fallback.kind` is `no_grammar`. | | `binary` | boolean? | A binary file has no `hunks` and no `gap_after`. | | `additions` | number | Added lines. | | `deletions` | number | Removed lines. | @@ -51,37 +46,34 @@ a bump is the syntax `class`; treat unknown classes as plain text. | `new_missing_newline` | boolean? | The new file's last line has no newline. | | `fallback` | Fallback? | Present when the file got a line diff instead of a semantic one. | | `hunks` | Hunk[]? | In file order. | -| `gap_after` | Gap? | Unchanged lines after the last hunk. For a file with no hunks (unchanged, or renamed without edits) this is the whole file. | +| `gap_after` | Gap? | Unchanged lines after the last hunk; the whole file when there are no hunks (unchanged, or renamed without edits). | | `outline` | OutlineItem[]? | Items the semantic diff found changed. | | `moves` | Move[]? | Code the semantic diff found moved. | | `renames` | Rename[]? | Identifiers the semantic diff found renamed. | -A file renders as: for each hunk, its `gap_before` (if any), its header, -its rows; then `gap_after` (if any). +Render order: for each hunk, `gap_before`, header, rows; then `gap_after`. ### Fallback | Field | Type | | | --- | --- | --- | -| `kind` | `"no_grammar"` \| `"grammar_unavailable"` \| `"semantic"` \| `"request_budget"` | `no_grammar`: no language is known for the path. `grammar_unavailable`: the path names a language whose grammar could not load. `semantic`: the semantic diff gave up, see `reason`. `request_budget`: the request's deadline passed before this file was diffed. | -| `reason` | `"too_large"` \| `"parse_errors"` \| `"error_coverage"` \| `"unparsed"` \| `"budget"` \| `"timeout"`? | Present when `kind` is `semantic`. The file was too large; had too many parse errors; parse errors covered too much of it; the parser gave up; the work budget ran out; the time budget ran out. | +| `kind` | `"no_grammar"` \| `"grammar_unavailable"` \| `"semantic"` \| `"request_budget"` | `no_grammar`: no language for the path. `grammar_unavailable`: the language's grammar failed to load. `semantic`: the semantic diff gave up, see `reason`. `request_budget`: the request's deadline passed before this file was diffed. | +| `reason` | `"too_large"` \| `"parse_errors"` \| `"error_coverage"` \| `"unparsed"` \| `"budget"` \| `"timeout"`? | Present when `kind` is `semantic`, in order: file too large; too many parse errors; parse errors cover too much; parser gave up; work budget ran out; time budget ran out. | | `message` | string | For people, e.g. `"exceeded the semantic time budget"`. Wording may change; branch on `kind` and `reason`. | -Line-diff files still get changed-word spans on `modified` rows. - ## Hunk | Field | Type | | | --- | --- | --- | -| `old_start`, `old_len`, `new_start`, `new_len` | number | The numbers of a unified diff's `@@ -old_start,old_len +new_start,new_len @@` line. A start is the side's first line in the hunk, or the line before it when that side is empty (`0` at the top of the file). | -| `header` | string? | The innermost item whose lines contain the hunk's first changed line (blank lines skipped), for the hunk's header line: a changed item's name from the semantic outline (`render_preview`), else a scope label from the grammar (`fn render_preview`, `impl Parser`, `class Parser`, `parse()`). Absent when nothing contains the change, as for a top-level statement between functions. | +| `old_start`, `old_len`, `new_start`, `new_len` | number | The `@@ -old_start,old_len +new_start,new_len @@` numbers. An empty side's start is the line before the hunk (`0` at the top of the file). | +| `header` | string? | The innermost item containing the hunk's first non-blank changed line: a changed item's name from the outline (`render_preview`), else a grammar scope label (`fn render_preview`, `impl Parser`, `parse()`). Absent when nothing contains it, e.g. a top-level statement between functions. | | `gap_before` | Gap? | Unchanged lines between the previous hunk (or the top of the file) and this one. | | `rows` | Row[] | In display order; each side's line numbers ascend. | ## Gap -Unchanged lines left out of the hunks. The lines are identical on both -sides; the starts differ when earlier changes shifted lines. +Unchanged lines left out of the hunks, identical on both sides; the starts +differ when earlier changes shifted lines. | Field | Type | | | --- | --- | --- | @@ -89,16 +81,15 @@ sides; the starts differ when earlier changes shifted lines. | `new_start` | number | First line on the new side. | | `len` | number | Lines in the gap, at least 1. | -To expand, send the server a Gap: the whole gap, or any sub-range of it -(the first 20 lines: `{old_start, new_start, len: 20}`; the last 20: -`{old_start: old_start + len - 20, new_start: new_start + len - 20, len: -20}`). The server answers with `context` rows (below), with syntax spans -when the request asked for syntax. The answer can be shorter than `len` -when the range runs past the end of the file or no longer matches it. +To expand, send a Gap: the whole gap or any sub-range (first 20 lines: +`{old_start, new_start, len: 20}`; last 20: `{old_start: old_start + len - 20, +new_start: new_start + len - 20, len: 20}`). The reply is `context` rows, with +syntax spans if syntax was requested. It can be shorter than `len` when the +range runs past the end of the file or no longer matches it. ## Row -A row is one of four kinds, told apart by `kind`. +Tagged by `kind`. ### `context` @@ -120,11 +111,11 @@ An unchanged line. | `added` | `new`: Line | | `modified` | `old`: Line, `new`: Line — a removed line beside the added line it became. | -Split view: `old` goes left, `new` right; a `removed` row leaves the right -empty and an `added` row the left. Unified view: a `modified` row is its -`old` line as removed then its `new` line as added; to read like `git -diff`, show a run of consecutive `modified` rows as all their old lines, -then all their new lines. +- Split view: `old` left, `new` right; `removed` leaves the right empty, + `added` the left. +- Unified view: a `modified` row is its `old` line removed, then its `new` line + added. To read like `git diff`, show a run of `modified` rows as all their + old lines, then all their new lines. ### Line @@ -137,39 +128,40 @@ One side of a changed row. | `changes` | ChangeSpan[]? | What changed within the line. Sorted, never overlapping. | | `syntax` | SyntaxSpan[]? | From this side's file. | -A `removed` or `added` line without `changes` changed as a whole. A -`modified` line without `changes` has no word-level detail: the lines are -too different to compare word by word, too long, or differ only in what -`text` does not show (a `\r`, an invalid byte, the final newline). +Without `changes`, a `removed` or `added` line changed as a whole; a `modified` +line has no word-level detail (too different, too long, or differing only in +what `text` hides: a `\r`, an invalid byte, the final newline). ### ChangeSpan | Field | Type | | | --- | --- | --- | | `start`, `end` | number | UTF-16 offsets within the line's `text`. | -| `intensity` | `"novel"` \| `"novel_word"` \| `"unchanged_context"` \| `"renamed"` | `novel`: syntax that exists only on this side (an inserted or deleted expression); `novel_word`: a changed word in an otherwise matched line; `unchanged_context`: unchanged syntax inside a changed line, for a subdued highlight or none; `renamed`: an identifier matched as renamed (see `renames`). | +| `intensity` | `"novel"` \| `"novel_word"` \| `"unchanged_context"` \| `"renamed"` | `novel`: syntax only on this side (an inserted or deleted expression). `novel_word`: a changed word in an otherwise matched line. `unchanged_context`: unchanged syntax inside a changed line; subdue or skip it. `renamed`: an identifier matched as renamed (see `renames`). | -Line-diff files only use `novel_word`. +Line-diff files get only `novel_word` spans, on `modified` rows. ### SyntaxSpan -Present only when the request asked for syntax, the file's language has a -grammar, and the side is not too large. Text outside every span is plain. +Present only when syntax was requested, the language has a grammar, and the +side is under the size cap. Text outside every span is plain. | Field | Type | | | --- | --- | --- | | `start`, `end` | number | UTF-16 offsets within the line's `text`. Sorted, never overlapping. | -| `class` | string | One of `keyword`, `string`, `comment`, `number`, `type`, `function`, `operator`, `punctuation`, `variable`, `constant`, `builtin`, `attribute`, `tag`, `property`, `namespace`, `label`, `preprocessor`. Map each to a theme colour; treat any other value as plain. | +| `class` | string | One of `keyword`, `string`, `comment`, `number`, `type`, `function`, `operator`, `punctuation`, `variable`, `constant`, `builtin`, `attribute`, `tag`, `property`, `namespace`, `label`, `preprocessor`. Map each to a theme colour; render any other value as plain. | -Syntax comes from parsing each whole file, so a line inside a multi-line -string or comment is highlighted as such. Changes and syntax are -independent layers: draw the change background, then the syntax colour. +Each whole file is parsed, so lines inside multi-line strings and comments +highlight correctly. Changes and syntax are independent layers: draw the change +background, then the syntax colour. ## OutlineItem +New-side items first, then old-side ones. + | Field | Type | | | --- | --- | --- | -| `side` | `"old"` \| `"new"` | Which file `line` refers to: `removed` items are on the old side, the rest on the new. | +| `side` | `"old"` \| `"new"` | Which file `line` refers to: `removed` items are old-side, the rest new-side. | | `line` | number | The item's first line. | | `end_line` | number? | The item's last line, inclusive. | | `kind` | string | The grammar's node kind, e.g. `function_item`, `class_declaration`. | @@ -177,12 +169,10 @@ independent layers: draw the change background, then the syntax colour. | `change` | `"added"` \| `"removed"` \| `"renamed"` \| `"moved"` \| `"edited"` | `renamed`: only identifiers inside it were renamed. | | `edits` | number? | Changed lines of an `edited` item. | -New-side items come first, then old-side ones. - ## Move -`old_len` lines starting at old line `old_start` moved to `new_len` lines -starting at new line `new_start`. +`old_len` lines from old line `old_start` moved to `new_len` lines at new line +`new_start`. | Field | Type | | | --- | --- | --- | @@ -280,10 +270,9 @@ type Rename = { old: string; new: string; uses: number }; ## Example -A Rust file where `render` became `render_preview` and an argument -changed after a line with non-ASCII text, diffed semantically, without -syntax. Lines 1–9 and 14–20 are unchanged and collapsed. `é` is one UTF-16 -unit and `😀` two, so the changed `2` starts at 16 (byte 19). +A semantic Rust diff without syntax: `render` became `render_preview`, and an +argument changed after non-ASCII text. Lines 1–9 and 14–20 are collapsed. `é` +is one UTF-16 unit and `😀` two, so the changed `2` starts at 16 (byte 19). ```json { @@ -371,8 +360,7 @@ unit and `😀` two, so the changed `2` starts at 16 (byte 19). } ``` -With syntax requested, rows also carry spans, e.g. the old line 12 above -gains: +With syntax requested, old line 12 above also gains: ```json "syntax": [ diff --git a/crates/diffy-compare/src/document.rs b/crates/diffy-compare/src/document.rs index d074a953..b9df603c 100644 --- a/crates/diffy-compare/src/document.rs +++ b/crates/diffy-compare/src/document.rs @@ -140,8 +140,8 @@ pub fn clean_spans(spans: Vec, text: &str) -> Vec { out } -/// Builds one file from semantic rows (cleaned, see [`clean_spans`]): -/// rows become hunks, and lines between rows that differ are line-diffed. +/// One file from cleaned (see [`clean_spans`]) semantic rows; differing lines +/// between rows are line-diffed. pub fn document_from_rows( meta: FileMeta, old: TextStore, diff --git a/crates/diffy-compare/src/file.rs b/crates/diffy-compare/src/file.rs index 96b679ca..533d8da8 100644 --- a/crates/diffy-compare/src/file.rs +++ b/crates/diffy-compare/src/file.rs @@ -23,7 +23,7 @@ pub enum Prepared { }, } -/// Settles the files no diff algorithm needs to look at. +/// Settles files that need no diff. pub fn prepare(input: &FileInput) -> Result { let meta = input.meta(); if is_binary(&input.old) || is_binary(&input.new) { @@ -72,8 +72,7 @@ pub(crate) fn diff_one( let Some(grammar) = grammars(&tag) else { return line(meta, Fallback::GrammarUnavailable(tag.to_string())); }; - // Loading the grammar can take a while the first time, so the budget - // is measured again after it. + // A first grammar load can be slow; re-check the budget. let Some(max_time) = budget(options) else { return line(meta, Fallback::RequestBudget); }; @@ -85,8 +84,7 @@ pub(crate) fn diff_one( let Some(injector) = &grammar.injector else { return Vec::new(); }; - // Regions repeat a few languages many times (every `{expression}` - // of a Svelte template), so each is looked up once. + // Regions repeat a few languages many times; look each up once. let mut seen: Vec<(LanguageId, Option)> = Vec::new(); let mut out = Vec::new(); for (language, range) in injector.find(tree, source) { diff --git a/crates/diffy-compare/src/lib.rs b/crates/diffy-compare/src/lib.rs index 8b2252ac..ac814eb7 100644 --- a/crates/diffy-compare/src/lib.rs +++ b/crates/diffy-compare/src/lib.rs @@ -1,20 +1,9 @@ -//! Headless file comparison: the chain from a changed file's bytes to a -//! [`DiffDocument`], with no UI, Git, or network dependencies. +//! Headless file comparison: changed files' bytes to [`DiffDocument`]s, with +//! no UI, Git, or network dependencies. //! -//! For each file the caller hands over its status, old and new paths, and -//! old and new bytes ([`FileInput`]). [`diff_files`] picks a language from -//! the path, loads its grammar from the caller's [`GrammarStore`], and runs -//! the semantic diff, falling back to a line diff (and saying why, see -//! [`Fallback`]) when no grammar exists or the semantic diff gives up. -//! Files are diffed on a few worker threads. -//! -//! Budgets: each file's semantic diff gets [`Options::file_budget`]; a -//! request may also set a [`Options::deadline`], after which remaining -//! files get line diffs only. [`Options::cancel`] stops the request -//! between files with [`Error::Cancelled`]. -//! -//! With the `serde` feature, [`wire`] projects results into a versioned -//! display format that non-Rust clients render from. +//! [`diff_files`] runs the semantic diff with the path's grammar, falling back +//! to a line diff ([`Fallback`]) when there is none or the semantic diff gives +//! up. The `serde` feature adds [`wire`], a versioned display format. use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -40,10 +29,10 @@ pub use file::{Prepared, prepare}; pub use grammar::{SemanticGrammar, semantic_grammar}; pub use line::{CONTEXT_LINES, binary_document, line_diff}; -/// The semantic diff's default time budget per file. +/// Default [`Options::file_budget`]. pub const FILE_BUDGET: Duration = Duration::from_millis(400); -/// One changed file, as the caller read it. +/// One changed file. #[derive(Debug, Clone, Default)] pub struct FileInput { pub status: FileStatus, @@ -64,7 +53,7 @@ impl FileInput { .unwrap_or_default() } - /// The document metadata for this file: paths and status. + /// Paths and status as document metadata. pub fn meta(&self) -> FileMeta { FileMeta { old_path: self.old_path.as_deref().map(Arc::from), @@ -80,10 +69,9 @@ impl FileInput { pub enum Fallback { /// No language is known for the path. NoGrammar, - /// The path names a language (this tag) whose grammar the store - /// cannot load. + /// The store cannot load the grammar for this language tag. GrammarUnavailable(String), - /// The semantic diff gave up: size, parse errors, work or time budget. + /// The semantic diff gave up. Semantic(diffy_semantic::Fallback), /// The request's deadline passed before this file was diffed. RequestBudget, @@ -100,8 +88,8 @@ impl std::fmt::Display for Fallback { } } -/// One file's result. Binary files, added and deleted files, and unchanged -/// files never run the semantic diff and have no fallback. +/// One file's result. Binary, added, deleted and unchanged files skip the +/// semantic diff and have no fallback. #[derive(Debug, Clone)] pub struct FileDiff { /// Holds exactly one file. @@ -109,11 +97,9 @@ pub struct FileDiff { pub fallback: Option, } -/// Hooks for callers that track progress or account for work per file. -/// Called from worker threads. +/// Per-file hooks, called from worker threads. pub trait Observer: Sync { - /// Runs `work`, which diffs the file at `path`; wrap it to time or - /// attribute the work. The default just runs it. + /// Runs `work`, the diff of `path`; wrap it to time or attribute it. fn file(&self, path: &str, work: &mut dyn FnMut()) { let _ = path; work(); @@ -129,7 +115,7 @@ pub struct Options<'a> { /// Grammars for the semantic diff; [`GrammarStore::none`] gives line /// diffs only. pub grammars: &'a GrammarStore, - /// Time budget for one file's semantic diff. + /// Per-file semantic diff budget. pub file_budget: Duration, /// When the whole request must be done. Files started after it get /// line diffs ([`Fallback::RequestBudget`]); a file's semantic budget @@ -137,8 +123,7 @@ pub struct Options<'a> { pub deadline: Option, /// Set to stop the request; checked between files. pub cancel: Option<&'a AtomicBool>, - /// Confirms candidate renames of identifiers, see - /// [`diffy_semantic::Vet`]. + /// Confirms identifier renames; see [`diffy_semantic::Vet`]. pub vet: Option<&'a diffy_semantic::Vet<'a>>, /// Worker threads; 0 picks from the file count and the machine. pub workers: usize, @@ -163,7 +148,7 @@ impl<'a> Options<'a> { pub enum Error { /// [`Options::cancel`] was set. Cancelled, - /// A document failed to build; a bug, never expected. + /// A document failed to build (a bug). Build(String), /// A worker thread could not start or panicked. Worker(String), @@ -190,8 +175,7 @@ pub fn diff_file(input: &FileInput, options: &Options<'_>) -> Result { }) } -/// [`diff_file`] with grammars from `grammars` instead of -/// [`Options::grammars`], for tools that wait on or swap grammars. +/// [`diff_file`] with a custom grammar lookup. #[doc(hidden)] pub fn diff_file_with( input: &FileInput, @@ -208,8 +192,7 @@ pub fn diff_file_with( Ok(diff) } -/// Diffs every file, in parallel when there are several, and returns the -/// results in input order. +/// Diffs files in parallel; results are in input order. pub fn diff_files(inputs: &[FileInput], options: &Options<'_>) -> Result> { run::diff_all(inputs, options, &|tag| { semantic_grammar(options.grammars, tag) diff --git a/crates/diffy-compare/src/line.rs b/crates/diffy-compare/src/line.rs index 2fd3e606..eb41c23c 100644 --- a/crates/diffy-compare/src/line.rs +++ b/crates/diffy-compare/src/line.rs @@ -1,4 +1,4 @@ -//! Line diffs: what a file gets when the semantic diff does not run. +//! Line diffs. use diffy_diff::{ DiffDocument, DocumentBuilder, FileMeta, SourceCoverage, TextStore, diff_from_changes, @@ -24,7 +24,7 @@ pub fn line_diff(mut meta: FileMeta, old: &[u8], new: &[u8]) -> Result Result { meta.binary = true; let mut builder = DocumentBuilder::new(); @@ -40,8 +40,7 @@ pub fn binary_document(mut meta: FileMeta) -> Result { Ok(builder.build()) } -/// Histogram line diff with Git's slider heuristics, as `git diff` and -/// gix compute it. +/// Histogram with Git's slider heuristics, matching `git diff`. pub(crate) fn line_changes(old: &[u8], new: &[u8]) -> Vec { let input = InternedInput::new(old, new); let mut diff = Diff::compute(Algorithm::Histogram, &input); diff --git a/crates/diffy-compare/src/run.rs b/crates/diffy-compare/src/run.rs index b3878675..80948c15 100644 --- a/crates/diffy-compare/src/run.rs +++ b/crates/diffy-compare/src/run.rs @@ -22,9 +22,7 @@ pub(crate) fn cancelled(options: &Options<'_>) -> bool { .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) } -/// Diffs `input` under the observer's [`Observer::file`] hook. -/// -/// [`Observer::file`]: crate::Observer::file +/// Diffs `input` inside [`crate::Observer::file`]. pub(crate) fn observed( input: &FileInput, options: &Options<'_>, @@ -39,13 +37,12 @@ pub(crate) fn observed( }); match out { Some(out) => out, - // The observer did not run the work; it still has to happen. + // The observer skipped `work`; run it anyway. None => diff_one(input, options, grammars), } } -/// Diffs `inputs` on worker threads, which take the next file in order -/// until none remain or the request is cancelled. +/// Diffs `inputs` on workers that take files in order until done or cancelled. pub(crate) fn diff_all( inputs: &[FileInput], options: &Options<'_>, @@ -101,10 +98,9 @@ pub(crate) fn diff_all( done.into_iter().map(|(_, diff)| diff).collect() } -/// Runs `work` for every index in `0..len` on worker threads and returns -/// the results in index order. The calling thread works too, so a worker -/// that cannot start only costs parallelism; a worker's panic is resumed -/// on the caller. +/// Runs `work` for `0..len` on workers; results are in index order. The caller +/// works too, so a worker that fails to spawn only costs parallelism; worker +/// panics resume on the caller. #[cfg(feature = "serde")] pub(crate) fn map_parallel(len: usize, work: &(dyn Fn(usize) -> T + Sync)) -> Vec { let next = AtomicUsize::new(0); diff --git a/crates/diffy-compare/src/wire.rs b/crates/diffy-compare/src/wire.rs index 8eeb6451..02d147ce 100644 --- a/crates/diffy-compare/src/wire.rs +++ b/crates/diffy-compare/src/wire.rs @@ -1,25 +1,9 @@ -//! The versioned display format that non-Rust clients render diffs from. +//! The versioned display format non-Rust clients render diffs from. //! -//! [`file()`] and [`document`] project compare results ([`FileDiff`]) into -//! plain serializable values; [`expand`] produces the rows of a collapsed -//! gap on request. `FORMAT.md` beside this crate's manifest is the -//! client-facing spec of every field. -//! -//! Versioning: [`Document::version`] is [`FORMAT_VERSION`]. It is bumped -//! for any breaking change: removing or renaming a field or enum value, -//! changing a field's type, meaning, or coordinates, or making an optional -//! field required. Adding an optional field, or a value to an enum that -//! clients are told to tolerate (syntax classes), is not breaking and -//! keeps the version. -//! -//! Coordinates: line numbers are one-based. Span offsets are UTF-16 code -//! units within the row's text, the way JavaScript indexes strings, and -//! ranges are half-open. Text is each line as [`TextStore::display_line`] -//! shows it: no newline, no trailing `\r`, invalid UTF-8 bytes as U+FFFD. -//! -//! Rows are layout-neutral: a `modified` row carries the removed line and -//! the added line it became, so split and unified layouts both render -//! without recomputing anything. +//! [`file()`] and [`document`] project [`FileDiff`]s; [`expand`] returns the +//! rows of a collapsed gap. The crate's `FORMAT.md` specifies every field. +//! Lines are one-based; span offsets are half-open UTF-16 code units into +//! [`TextStore::display_line`] text. use std::borrow::Cow; use std::cell::OnceCell; @@ -36,23 +20,21 @@ use serde::{Deserialize, Serialize}; use crate::{FileDiff, FileInput}; -/// The format version every [`Document`] carries. +/// Bump on any breaking change, as `FORMAT.md` defines it. pub const FORMAT_VERSION: u32 = 1; -/// Sides longer than this get no syntax spans or scope headers by default. +/// Default [`WireOptions::syntax_max_bytes`]. pub const SYNTAX_MAX_BYTES: usize = 1 << 20; /// What a projection adds beyond the diff itself. #[derive(Debug, Clone, Copy)] pub struct WireOptions<'a> { - /// Grammars for syntax spans and for hunk headers of files without a - /// semantic outline. `None` (the default) emits neither. + /// Grammars for syntax spans and scope headers; `None` emits neither. pub syntax: Option<&'a GrammarStore>, /// Sides longer than this, in bytes, get no syntax spans or scope /// headers. pub syntax_max_bytes: usize, - /// Files projected after this get no syntax spans or scope headers, so - /// highlighting cannot run a request past its budget. + /// Files projected after this get no syntax spans or scope headers. pub deadline: Option, /// Once set, the remaining files get no syntax spans or scope headers. pub cancel: Option<&'a AtomicBool>, @@ -87,9 +69,8 @@ pub struct File { /// `None` for a deleted file. #[serde(default, skip_serializing_if = "Option::is_none")] pub new_path: Option, - /// The language tag the path names (`rs`, `ts`, `dockerfile`), as - /// diffy names grammars; `None` when it names none or the compare - /// found no grammar for it. + /// The path's language tag (`rs`, `dockerfile`); `None` with + /// [`FallbackKind::NoGrammar`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub language: Option, /// A binary file has no hunks or gaps. @@ -112,8 +93,7 @@ pub struct File { /// whole file. #[serde(default, skip_serializing_if = "Option::is_none")] pub gap_after: Option, - /// Items the semantic diff found changed: new-side items, then - /// old-side ones. + /// New-side items, then old-side ones. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub outline: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -204,14 +184,13 @@ impl From<&crate::Fallback> for Fallback { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Hunk { - /// The `@@ -old_start,old_len +new_start,new_len @@` numbers: a start - /// is the first line, or the line before an empty side. + /// The `@@ -old_start,old_len +new_start,new_len @@` numbers; an empty + /// side's start is the line before the hunk. pub old_start: u32, pub old_len: u32, pub new_start: u32, pub new_len: u32, - /// The item enclosing the hunk's first change (`render_preview`, or a - /// scope label such as `impl Parser`). + /// The innermost item around the first change, e.g. `impl Parser`. #[serde(default, skip_serializing_if = "Option::is_none")] pub header: Option, /// Unchanged lines between the previous hunk (or the file start) and @@ -300,9 +279,8 @@ impl From for Intensity { } } -/// A highlighted run of a line. Offsets are UTF-16 code units within the -/// line's text; spans are sorted and disjoint, and text outside them is -/// plain. +/// A highlighted run of a line, with [`ChangeSpan`]'s offsets and ordering. +/// Text outside spans is plain. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SyntaxSpan { pub start: u32, @@ -392,8 +370,7 @@ fn is_false(value: &bool) -> bool { /// entries of the longer slice are ignored. pub fn document(inputs: &[FileInput], diffs: &[FileDiff], options: &WireOptions<'_>) -> Document { let len = inputs.len().min(diffs.len()); - // Highlighting parses both sides of every file, so files are projected - // on workers like the diff itself. + // Highlighting parses every file, so it runs on workers. let files = if options.syntax.is_some() { crate::run::map_parallel(len, &|at| file(&inputs[at], &diffs[at], options)) } else { @@ -407,8 +384,7 @@ pub fn document(inputs: &[FileInput], diffs: &[FileDiff], options: &WireOptions< } } -/// Projects one file. Highlighting and scope headers parse each side once; -/// everything else is linear in the rows shown. +/// Projects one file. pub fn file(input: &FileInput, diff: &FileDiff, options: &WireOptions<'_>) -> File { let late = options .deadline @@ -474,11 +450,9 @@ pub fn file(input: &FileInput, diff: &FileDiff, options: &WireOptions<'_>) -> Fi out } -/// Context rows for `gap` (or any sub-range of a gap) of `input`, with -/// the new side's syntax spans when `options` asks for them. Highlighting -/// parses the whole new side so lexical state is right. Rows stop early -/// where the range leaves either side or the sides differ, as a stale -/// request would. +/// Context rows for `gap`, with new-side syntax when `options` asks for it. +/// Highlighting parses the whole new side so lexical state is right. Stops +/// early where the range leaves either side or the sides differ (stale). pub fn expand(input: &FileInput, gap: &Gap, options: &WireOptions<'_>) -> Vec { if gap.len == 0 || gap.old_start == 0 @@ -667,8 +641,7 @@ impl<'a> Projector<'a> { let old_spans = self.doc.semantic_spans(0, DocSide::Old, old.1); let new_spans = self.doc.semantic_spans(0, DocSide::New, new.1); if old_spans.is_empty() && new_spans.is_empty() { - // A line-diff pair, or lines the semantic diff left - // unmarked: changed words come from the word diff. + // Unmarked by the semantic diff: use the word diff. let old_text = self.display(DocSide::Old, old.0); let new_text = self.display(DocSide::New, new.0); let words = inline_diff(old_text, new_text); @@ -750,9 +723,7 @@ impl<'a> Projector<'a> { // change between the anchor line and the one before. let (old, new) = match first_change { Some(bi) => { - // Blank lines lead most added and removed items and belong - // to the scope around them, so the first one with text - // anchors the change. + // Leading blank lines belong to the enclosing scope; skip them. let first_text = |side: DocSide, store: u32, len: u32| { (0..len) .find(|&k| { @@ -774,7 +745,7 @@ impl<'a> Projector<'a> { (h.new_start[hi].saturating_sub(1), true), ), }; - // The change shows on the new side when it has lines there. + // Prefer the new side when it has changed lines. let (side, other) = if new.1 { (DocSide::New, DocSide::Old) } else { @@ -785,9 +756,9 @@ impl<'a> Projector<'a> { DocSide::New => new, }; if let Some(semantics) = self.doc.semantics(0) { - // Edited items sit on the new side and removed ones on the - // old, so a side with no containing item defers to the other. - // Items on one side nest, so the shortest is the innermost. + // Edited items are new-side and removed ones old-side, so a side + // with no containing item defers to the other. Items on a side + // nest, so the shortest is innermost. let innermost = |side: DocSide| { let (line, lines) = anchor(side); semantics @@ -868,8 +839,8 @@ enum Step { Added(u32), } -/// Pairs a change block's lines as the semantic diff aligned them, when -/// its rows cover the block in order; offsets within the block. +/// Pairs a change block's lines as the semantic diff aligned them; `false` +/// unless the rows cover the block in order. fn structural_steps( rows: &[AlignedRow], at: &BlockAt, @@ -927,7 +898,6 @@ fn semantic( }) } -/// A context row of store line `store` of `text`, with its syntax. fn context_row( text: &TextStore, syntax: &[HighlightSpan], @@ -945,8 +915,7 @@ fn context_row( } } -/// Highlights a whole side once, when `options` asks for syntax and the -/// side is within the size cap. +/// Highlights a whole side, if `options` asks and it is within the size cap. fn highlights( options: &WireOptions<'_>, path: Option<&str>, diff --git a/crates/diffy-compare/src/wire/tests.rs b/crates/diffy-compare/src/wire/tests.rs index d8d867ea..f9ae0d16 100644 --- a/crates/diffy-compare/src/wire/tests.rs +++ b/crates/diffy-compare/src/wire/tests.rs @@ -241,9 +241,8 @@ fn invalid_utf8_lines_show_replacement_characters() { ); } -/// A structural document: `render` renamed to `render_preview` with an -/// argument changed after multibyte text, plus an outline, a move, and a -/// rename. +/// `render` renamed to `render_preview`; an argument changed after multibyte +/// text. fn semantic_diff() -> (FileInput, FileDiff) { let old = "fn keep() {}\nfn render() {\n draw(\"é😀\", 1);\n}\n"; let new = "fn keep() {}\nfn render_preview() {\n draw(\"é😀\", 2);\n}\n"; @@ -457,8 +456,7 @@ fn hunk_headers_name_the_innermost_containing_item() { assert_eq!(header(nested(), (1, 1)).as_deref(), Some("inner")); // Only the old side has lines, so its items come first. assert_eq!(header(nested(), (1, 0)).as_deref(), Some("removed")); - // An old-only change without an old item takes the new item around - // the gap it leaves, which must hold the lines on both sides of it. + // Without an old item, the new item must surround the gap on both sides. let new_only = || { vec![ item(DocSide::New, 0..4, "outer"), @@ -525,7 +523,6 @@ fn aligned_rows_pair_lines_where_the_semantic_diff_put_them() { ("context", Some(4), Some(5)), ] ); - // Unmarked pairs still get the word diff. let (_, new_line) = modified(&rows[1]); assert_eq!(new_line.changes, [span(3, 4, Intensity::NovelWord)]); } @@ -567,7 +564,6 @@ fn gaps_track_each_side_and_expand_into_context_rows() { assert_eq!(rows[0], context(14, 16, "line 14")); assert_eq!(rows[16], context(30, 32, "line 30")); - // The bottom three lines of the gap above the hunk. let tail = Gap { old_start: 5, new_start: 5, @@ -583,14 +579,13 @@ fn gaps_track_each_side_and_expand_into_context_rows() { ] ); - // Past the end of the file: only the lines that exist. let past = Gap { old_start: 29, new_start: 31, len: 10, }; assert_eq!(expand(&input, &past, &options).len(), 2); - // Stale: the sides differ, so nothing is claimed unchanged. + // Stale: the sides differ. let stale = Gap { old_start: 14, new_start: 14, @@ -745,7 +740,6 @@ fn syntax_spans_are_classes_sliced_per_line_in_utf16() { assert_eq!(utf16_slice(&old_line.text, draw.start, draw.end), "draw"); assert_eq!(draw.class, "function"); - // Expanded context is highlighted with the whole file's parse. let gap = hunk.gap_before.expect("gap before"); let rows = expand(&input, &gap, &options); let Row::Context { syntax, .. } = &rows[0] else { @@ -778,8 +772,7 @@ fn the_spec_example_matches_the_types() { let value: serde_json::Value = serde_json::from_str(example).expect("example parses"); let document: Document = serde_json::from_value(value.clone()).expect("example is a Document"); assert_eq!(document.version, FORMAT_VERSION); - // Nothing in the example is unknown to the types or spelled - // differently from what they write. + // Round-trips exactly: no unknown or misspelled fields. assert_eq!(serde_json::to_value(&document).expect("serializes"), value); } @@ -795,8 +788,7 @@ fn aligned_rows_that_miss_lines_fall_back_to_positional_pairs() { TextStore::new(new), ); file_builder.structural(true).hunk(0, 0).change(2, 2); - // Only the first pair is aligned; the second line of each side is - // never mentioned. + // Only the first pair is aligned. file_builder.row(AlignedRow { old: Some(0), new: Some(0), @@ -836,8 +828,8 @@ fn files_projected_past_the_deadline_skip_syntax() { } } -/// The hunk headers of a real semantic diff of `old` to `new`, without -/// syntax, so they come from the outline alone; `None` without the pack. +/// Hunk headers of a semantic diff without syntax, so from the outline alone; +/// `None` without the pack. fn outline_headers(language: &str, path: &str, old: &str, new: &str) -> Option> { let store = diffy_syntax::testing::store_with(language)?; let input = FileInput { @@ -936,8 +928,7 @@ fn hunk_headers_pick_the_innermost_item_on_the_change_side() { let Some(headers) = outline_headers("rust", "src/lib.rs", old, &new) else { return; }; - // An edit inside a method names the method, not its impl; a deletion - // inside an edited method names it from the new side; a removed - // method names itself from the old side. + // Edited method, not its impl; a deletion inside an edited method (new + // side); a removed method (old side). assert_eq!(headers, ["render", "other", "removed"]); } diff --git a/crates/diffy-syntax/Cargo.toml b/crates/diffy-syntax/Cargo.toml index b90777eb..02d68d43 100644 --- a/crates/diffy-syntax/Cargo.toml +++ b/crates/diffy-syntax/Cargo.toml @@ -30,11 +30,9 @@ ureq = { version = "3", default-features = false, features = ["rustls"], optiona [features] default = [] -# The tree-sitter runtime: highlighting, injections, scopes, and symbols -# over the grammars a store gets from `bundled` or, with `engine`, packs. +# Tree-sitter runtime (highlighting, injections, scopes, symbols); no grammars. runtime = ["dep:tree-sitter", "dep:tree-sitter-language", "dep:tracing"] -# Pack loading from local directories: manifests, SHA-256 checks, and -# shared libraries. +# `runtime` plus pack loading from local directories. engine = [ "runtime", "dep:libloading", @@ -45,9 +43,7 @@ engine = [ ] # Downloading packs on demand from a signed index. download = ["engine", "dep:quark-update", "dep:ureq"] -# Grammars compiled in from the bundle `syntax-pack bundle` writes, named -# by $DIFFY_SYNTAX_BUNDLE at build time (empty, with a warning, without it). -# Needs no pack loading. +# Grammars compiled in from $DIFFY_SYNTAX_BUNDLE (`syntax-pack bundle`). bundled = ["runtime"] [dev-dependencies] diff --git a/crates/diffy-syntax/PACKS.md b/crates/diffy-syntax/PACKS.md index 35e3e1ee..d61e1843 100644 --- a/crates/diffy-syntax/PACKS.md +++ b/crates/diffy-syntax/PACKS.md @@ -2,8 +2,8 @@ Load tree-sitter grammars at runtime, from packs, to highlight diffs. -- No grammar is compiled into the app. The `bundled` feature compiles a - fixed set into other builds (see [Bundled grammars](#bundled-grammars)). +- No grammar is compiled into the app; other builds can use + [bundled grammars](#bundled-grammars). - A pack is a shared library exporting the tree-sitter language function, plus its highlight query, an optional injection query, and a manifest. - A `GrammarStore` feeds the diff view; an app adapts it to Quark's @@ -134,32 +134,22 @@ cargo run -p syntax-pack -- check # re-check built packs for thi ## Bundled grammars -For builds that must highlight with no packs and no download step, such -as a headless server. The Diffy app does not use it. +Compiled-in grammars, for builds without packs or downloads (such as a +headless server). The app does not use them. - Default set: C, C++, CSS, Go, HTML, JavaScript, JSON, Kotlin, Markdown (with `markdown_inline`), Python, Rust, Swift, TOML, TSX, TypeScript, YAML. -- `syntax-pack bundle` builds and checks each as a host pack (which adapts - its queries), then compiles the same verified sources into a static - library per grammar for the target and links them together once to - catch clashing symbols. -- Output, `//`: `lib/` (static libraries), `queries/`, - `bundle.tsv` (the table `build.rs` reads), `THIRD_PARTY_NOTICES.md`. -- Build diffy-syntax with `--features bundled` and `DIFFY_SYNTAX_BUNDLE` - set to the absolute ``; `build.rs` reads `//` and - links it. -- Without `DIFFY_SYNTAX_BUNDLE` the feature builds with an empty bundle and - a cargo warning. -- `bundled` needs only the `runtime` tier, so a bundled-only build links no - pack loader (`libloading`, `ring`, JSON manifests) and has no - `StoreConfig::local_packs`. Turn on `engine` too to also load packs. -- Bundled grammars get the same tags, queries, ABI check, and `catalog()` - entries as packs; nothing is loaded or hashed at run time. +- `syntax-pack bundle` writes `//`: `lib/` (static + libraries), `queries/`, `bundle.tsv`, `THIRD_PARTY_NOTICES.md`. +- Build with `--features bundled` and `DIFFY_SYNTAX_BUNDLE` set to the + absolute ``. Unset, the bundle is empty and cargo warns. +- `bundled` needs only `runtime`: no `libloading`, `ring`, JSON manifests, + or `StoreConfig::local_packs`. Add `engine` to also load packs. +- Same tags, queries, ABI check, and `catalog()` entries as packs. - `diffy_syntax::bundled::languages()` lists what a build has. -- Ship `THIRD_PARTY_NOTICES.md` with anything you distribute: it holds each - grammar's license, repositories, commits, and license text, which MIT - requires. +- Ship `THIRD_PARTY_NOTICES.md` with anything you distribute; the grammar + licenses require it. ```sh cargo run -p syntax-pack -- bundle --out "$PWD/target/syntax-bundle" # default set, this host diff --git a/crates/diffy-syntax/README.md b/crates/diffy-syntax/README.md index 5925ce9c..09316d9b 100644 --- a/crates/diffy-syntax/README.md +++ b/crates/diffy-syntax/README.md @@ -4,16 +4,15 @@ Tree-sitter syntax highlighting for [Diffy](../../README.md). No grammar is compiled in by default: each language is a pack (a shared library, its queries, and a manifest) that a `GrammarStore` loads from local pack directories the app trusts or, with the `download` feature, from a -signed index. Code renders plain until its grammar is available. Builds that -cannot load packs (a headless server) link a fixed set with `bundled`. +signed index. Code renders plain until its grammar is available. With +`bundled`, a fixed set is compiled in instead. -Features: `runtime` (the tree-sitter runtime: highlighting, injections, -scopes, symbols), `engine` (`runtime` plus local packs), `download` -(`engine` plus fetching packs over HTTPS with Ed25519-verified indexes), and -`bundled` (`runtime` plus grammars compiled in from `syntax-pack bundle`, -with no pack loading, `libloading`, `ring`, or JSON manifests). Without -`runtime` the crate has no dependencies and every lookup falls back to -plain text. +Features: `runtime` (tree-sitter highlighting, injections, scopes, +symbols), `engine` (`runtime` plus local packs), `download` (`engine` plus +fetching packs over HTTPS with Ed25519-verified indexes), and `bundled` +(`runtime` plus grammars from `syntax-pack bundle`; no pack loading). +Without `runtime` the crate has no dependencies and every lookup falls back +to plain text. `highlight` runs synchronously; `HighlightWorker` runs on a background thread and drops requests a newer generation superseded, which suits code diff --git a/crates/diffy-syntax/build.rs b/crates/diffy-syntax/build.rs index 43cebd19..6e69c085 100644 --- a/crates/diffy-syntax/build.rs +++ b/crates/diffy-syntax/build.rs @@ -1,10 +1,9 @@ use std::fmt::Write as _; use std::path::{Path, PathBuf}; -/// The directory `syntax-pack bundle --out` wrote, read by the `bundled` -/// feature. +/// The directory `syntax-pack bundle --out` wrote. const BUNDLE_VAR: &str = "DIFFY_SYNTAX_BUNDLE"; -/// First line of `//bundle.tsv`; `syntax-pack` writes it. +/// First line of `//bundle.tsv`. const HEADER: &str = "# diffy-syntax bundle 1"; const QUERIES: [&str; 5] = ["highlights", "injections", "locals", "semantic", "tags"]; @@ -22,8 +21,7 @@ fn main() { } } -/// Writes `$OUT_DIR/bundled.rs`, the table `src/bundled.rs` includes, and -/// links the bundle's static libraries. +/// Writes `$OUT_DIR/bundled.rs` for `src/bundled.rs` and links the bundle. fn bundle(target: &str) -> Result<(), String> { println!("cargo:rerun-if-env-changed={BUNDLE_VAR}"); let out_dir = std::env::var_os("OUT_DIR").ok_or("cargo did not set OUT_DIR")?; @@ -47,10 +45,9 @@ fn bundle(target: &str) -> Result<(), String> { std::fs::write(&out, code).map_err(|e| format!("{}: {e}", out.display())) } -/// The table for the bundle in `dir`, one line of `bundle.tsv` per grammar: -/// language, symbol, ABI, library, aliases, extensions, source SHA-256 -/// (lists comma-separated, `-` when empty). Queries are -/// `queries//.scm`. +/// The table for the bundle in `dir`. `bundle.tsv` columns: language, +/// symbol, ABI, library, aliases, extensions, source SHA-256 (lists +/// comma-separated, `-` when empty). Queries: `queries//.scm`. fn generate(dir: &Path, target: &str) -> Result { let manifest = dir.join("bundle.tsv"); let text = std::fs::read_to_string(&manifest).map_err(|e| { @@ -117,8 +114,7 @@ fn generate(dir: &Path, target: &str) -> Result { println!("cargo:rerun-if-changed={}", archive.display()); println!("cargo:rustc-link-lib=static={library}"); - // Each grammar gets its own Rust name, so only the link name comes - // from the bundle. + // Only the link name comes from the bundle; the Rust name is ours. let _ = writeln!( externs, " #[link_name = {symbol:?}]\n fn grammar_{index}() -> *const ();" diff --git a/crates/diffy-syntax/src/bundled.rs b/crates/diffy-syntax/src/bundled.rs index 0d4d15f6..029c379c 100644 --- a/crates/diffy-syntax/src/bundled.rs +++ b/crates/diffy-syntax/src/bundled.rs @@ -1,9 +1,6 @@ -//! Grammars compiled into the binary with the `bundled` feature. -//! -//! `syntax-pack bundle` compiles a fixed set of grammars from their pinned -//! sources into static libraries and writes their queries and a table; -//! `build.rs` links them when `DIFFY_SYNTAX_BUNDLE` names that directory. -//! Without it the bundle is empty. A store uses them only after +//! Grammars compiled in with the `bundled` feature, from the +//! `syntax-pack bundle` output `DIFFY_SYNTAX_BUNDLE` names at build time +//! (empty without it). Stores use them only after //! [`crate::StoreConfig::bundled`]. /// One compiled-in grammar, as `build.rs` generates it. @@ -11,7 +8,7 @@ pub(crate) struct Bundled { pub(crate) name: &'static str, pub(crate) aliases: &'static [&'static str], pub(crate) extensions: &'static [&'static str], - /// The grammar's tree-sitter ABI, checked against the linked grammar. + /// Declared tree-sitter ABI, checked against the linked grammar. pub(crate) abi: u32, pub(crate) language: unsafe extern "C" fn() -> *const (), /// SHA-256 of the pinned sources it was built from. @@ -24,7 +21,7 @@ pub(crate) struct Bundled { } impl Bundled { - /// The canonical name, aliases, and extensions, each a lookup key. + /// Lookup keys: name, aliases, extensions. pub(crate) fn tags(&self) -> impl Iterator { std::iter::once(self.name) .chain(self.aliases.iter().copied()) @@ -34,13 +31,12 @@ impl Bundled { include!(concat!(env!("OUT_DIR"), "/bundled.rs")); -/// The canonical names of the grammars compiled into this build; empty -/// when it was built without `DIFFY_SYNTAX_BUNDLE`. +/// Names of the compiled-in grammars; empty without `DIFFY_SYNTAX_BUNDLE`. pub fn languages() -> impl ExactSizeIterator + Clone { GRAMMARS.iter().map(|grammar| grammar.name) } -/// The bundled grammar a name, alias, or extension names. +/// By name, alias, or extension, ignoring ASCII case. pub(crate) fn find(tag: &str) -> Option<&'static Bundled> { GRAMMARS .iter() diff --git a/crates/diffy-syntax/src/engine.rs b/crates/diffy-syntax/src/engine.rs index 380b7141..4f6cfa03 100644 --- a/crates/diffy-syntax/src/engine.rs +++ b/crates/diffy-syntax/src/engine.rs @@ -33,8 +33,7 @@ pub fn supported_abi() -> std::ops::RangeInclusive { ts::MIN_COMPATIBLE_LANGUAGE_VERSION as u32..=ts::LANGUAGE_VERSION as u32 } -/// Why a grammar, from a pack or the bundle, did not compile. Packs report -/// it as the matching `pack::PackError`. +/// Why a grammar did not compile; packs map it to `pack::PackError`. #[derive(Debug)] pub(crate) enum GrammarError { Abi { abi: u32, min: u32, max: u32 }, @@ -131,12 +130,10 @@ impl Grammar { }) } - /// A grammar compiled into the binary, with its queries. #[cfg(feature = "bundled")] pub(crate) fn bundled(bundled: &crate::bundled::Bundled) -> Result { - // SAFETY: the function is the tree-sitter language function of a - // grammar `syntax-pack bundle` compiled from pinned, hashed sources - // and linked into this binary. + // SAFETY: a tree-sitter language function linked in from pinned, + // hashed sources by `syntax-pack bundle`. let language = unsafe { tree_sitter_language::LanguageFn::from_raw(bundled.language) }; let grammar = Self::compile( language, @@ -149,8 +146,7 @@ impl Grammar { locals: bundled.locals.map(Arc::from), semantic: bundled.semantic.map(Arc::from), tags: bundled.tags.map(Arc::from), - // Identifies the grammar build for tag caches, as a pack's - // library hash does. + // Keys tag caches, like a pack's library hash. library: Some(Arc::from(bundled.source)), ..grammar }) @@ -184,8 +180,7 @@ impl Grammar { self.tags.as_ref() } - /// Checks `language`'s ABI against the one its pack or bundle - /// declares and compiles its highlight and injection queries. + /// Checks the ABI against `expected_abi` and compiles the queries. fn compile( language: tree_sitter_language::LanguageFn, expected_abi: u32, diff --git a/crates/diffy-syntax/src/lib.rs b/crates/diffy-syntax/src/lib.rs index b87c9ccb..8074a1b2 100644 --- a/crates/diffy-syntax/src/lib.rs +++ b/crates/diffy-syntax/src/lib.rs @@ -6,10 +6,8 @@ //! queries and a manifest (see [`pack`]), from a [`GrammarStore`] the app //! configures: local pack directories it trusts, and with the `download` //! feature a signed index it downloads missing packs from. Until a -//! language's grammar is available its code renders plain. The exception -//! is the `bundled` feature, for builds that cannot load packs: it links -//! the grammars `syntax-pack bundle` compiled (see [`bundled`] and -//! PACKS.md), used after [`StoreConfig::bundled`]. +//! language's grammar is available its code renders plain. The `bundled` +//! feature links a fixed set instead (see [`bundled`]). //! //! [`highlight`] runs synchronously; [`HighlightWorker`] runs it on a //! background thread and drops requests superseded by a newer generation @@ -62,12 +60,10 @@ //! takes the process down; a panic in the highlighter is caught by the //! worker. Packs come from pinned, hashed grammar sources built in CI. //! - Bundled grammars are part of the binary, built from the same pinned, -//! hashed sources, so they are as trusted as the rest of it; nothing is -//! loaded or hashed at run time. +//! hashed sources; nothing is loaded at run time. -// `runtime` alone has no grammar source: it is for libraries that take a -// store from a crate turning on `bundled` or `engine`, which Cargo unifies -// in. Until then the code only a source reaches is dead. +// `runtime` alone has no grammar source; a dependent enabling `engine` or +// `bundled` supplies one through feature unification. #![cfg_attr( all(feature = "runtime", not(any(feature = "engine", feature = "bundled"))), allow(dead_code, unused_mut, unused_variables) diff --git a/crates/diffy-syntax/src/store.rs b/crates/diffy-syntax/src/store.rs index 79b5668e..ce330d21 100644 --- a/crates/diffy-syntax/src/store.rs +++ b/crates/diffy-syntax/src/store.rs @@ -577,8 +577,8 @@ impl StoreConfig { self } - /// Uses the grammars compiled into the binary ([`crate::bundled`]), - /// after the local pack roots and before any download. + /// Uses the compiled-in grammars ([`crate::bundled`]), searched after + /// local packs and before downloads. #[cfg(feature = "bundled")] pub fn bundled(mut self) -> Self { self.bundled = true; @@ -695,7 +695,7 @@ impl Inner { } fn ships_tags(&self, tag: &str) -> bool { - // Only pack sources look past the tags already resolved. + // Only pack sources mutate state here. #[cfg_attr(not(feature = "engine"), allow(unused_mut))] let mut state = lock(&self.state); if let Some(Tag::Ready(grammar)) = state.tags.get(tag) { diff --git a/crates/diffy-syntax/src/tests.rs b/crates/diffy-syntax/src/tests.rs index d6d4cbd5..374b7852 100644 --- a/crates/diffy-syntax/src/tests.rs +++ b/crates/diffy-syntax/src/tests.rs @@ -1541,10 +1541,9 @@ mod bundled { use super::*; use HighlightKind::*; - /// Each default bundled language: a tag that should reach it, its - /// sample (a file under tools/syntax-pack/samples, or inline source), - /// and text the sample must color with a kind, as `syntax-pack check` - /// expects of its captures. + /// Language, a tag that reaches it, sample (file in + /// tools/syntax-pack/samples, or inline source), and text that must get + /// each kind. type Case = ( &'static str, &'static str, @@ -1604,8 +1603,7 @@ mod bundled { "fun main() {\n val greeting = \"hi\"\n}\n", &[("fun", Keyword), ("\"hi\"", String)], ), - // Its inline spans come from markdown_inline and the fence from - // rust, both bundled, through its injection query. + // Inline spans and the fence arrive through injections. ( "markdown", "md", @@ -1680,10 +1678,8 @@ mod bundled { std::fs::read_to_string(&path).unwrap() } - // Catches a bundled grammar that is not linked, not reached by its - // tags, whose queries do not compile, or whose injections do not reach - // the other bundled grammars. Needs a bundle with the default set - // (`syntax-pack bundle`, then DIFFY_SYNTAX_BUNDLE). + // Needs a default-set bundle (`syntax-pack bundle`, then + // DIFFY_SYNTAX_BUNDLE). #[test] fn bundled_grammars_resolve_by_tag_and_highlight_their_samples() { if crate::bundled::languages().len() == 0 { @@ -1713,8 +1709,6 @@ mod bundled { assert_eq!(store.status(&rust), LanguageStatus::Ready); } - // Catches bundled grammars missing from the catalog, or a store that - // did not opt in seeing them. #[test] fn the_catalog_lists_bundled_grammars_only_when_the_store_opts_in() { if crate::bundled::languages().len() == 0 { diff --git a/src/core/compare/semantic.rs b/src/core/compare/semantic.rs index 37e4e22f..cd6662f4 100644 --- a/src/core/compare/semantic.rs +++ b/src/core/compare/semantic.rs @@ -184,8 +184,7 @@ fn rename_vet<'g>( } } -/// Attributes each file's diff to the energy monitor and reports -/// `LoadingFiles` progress every few files. +/// Per-file energy attribution and throttled `LoadingFiles` progress. struct LoadingFiles<'a> { reporter: Option<&'a dyn ProgressSink>, } diff --git a/src/core/error.rs b/src/core/error.rs index 3863c19e..3e03cec0 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -143,7 +143,7 @@ impl DiffyError { impl From for DiffyError { fn from(error: diffy_compare::Error) -> Self { match error { - // Keeps the message document builds have always surfaced. + // Build failures stay parse errors, as the app reports them. diffy_compare::Error::Build(error) => Self::Parse(error), error => Self::General(error.to_string()), } diff --git a/tools/syntax-pack/README.md b/tools/syntax-pack/README.md index 89d034c1..33d2991b 100644 --- a/tools/syntax-pack/README.md +++ b/tools/syntax-pack/README.md @@ -23,22 +23,21 @@ subcommand. ## Bundles `bundle` compiles grammars into static libraries for diffy-syntax's -`bundled` feature; usage is in the -[guide](../../crates/diffy-syntax/PACKS.md#bundled-grammars). - -- Each language is first built and checked as a pack for the host, from the - same fetched and hash-checked sources, so its queries are adapted even - for another target. -- Static libraries are compiled with `-fvisibility=hidden` and PIC, and - with the dynamic CRT (`/MD`) on MSVC, as Rust uses by default. -- All libraries are then linked into one throwaway shared library for the - target, so two grammars defining one global symbol fail the bundle. -- C++ scanners are refused: every consumer would need the C++ runtime. -- `THIRD_PARTY_NOTICES.md` copies the license files at the top of each - grammar repository (required) and any query repository (the - nvim-treesitter-queries-* ones have none). -- A cross-target bundle needs the target's C compiler and archiver as - `build` does (`CC_`, `AR_`), plus the host's. +`bundled` feature ([usage](../../crates/diffy-syntax/PACKS.md#bundled-grammars)). + +- Each language is first built and checked as a host pack from the same + sources, which adapts its queries for any target. +- Static libraries: `-fvisibility=hidden`, PIC, and `/MD` on MSVC (Rust's + default CRT). +- All libraries are linked into one throwaway shared library for the + target, so a global symbol two grammars define fails the bundle. +- C++ scanners are refused: consumers would need the C++ runtime. +- `THIRD_PARTY_NOTICES.md` copies the top-level license files of each + grammar repository (required) and query repository; + nvim-treesitter-queries-* repositories have none, so nvim-treesitter's + Apache-2.0 text is appended. +- Cross-target bundles need `CC_` and `AR_`, as `build` + does, plus the host compiler. ```sh # Per target, on any runner with that target's C toolchain: diff --git a/tools/syntax-pack/src/main.rs b/tools/syntax-pack/src/main.rs index 64514895..e96aa70a 100644 --- a/tools/syntax-pack/src/main.rs +++ b/tools/syntax-pack/src/main.rs @@ -12,12 +12,11 @@ //! //skipped.json; a grammars.toml language fails the build. //! Grammars marked `generate` need tree-sitter CLI 0.26.8 ($TREE_SITTER). //! syntax-pack bundle [LANGUAGE...] [--target TRIPLE] [--out DIR] [--sources DIR] [--work DIR] [--jobs N] -//! Compile LANGUAGEs (default: BUNDLE_DEFAULT) from their pinned, -//! hashed sources into static libraries for TRIPLE, for diffy-syntax's -//! `bundled` feature (DIFFY_SYNTAX_BUNDLE=DIR). Each is first built and -//! checked as a pack for this host, which adapts its queries. Writes -//! //: lib/, queries//, bundle.tsv, and -//! THIRD_PARTY_NOTICES.md. --out defaults to /target/syntax-bundle. +//! Compile LANGUAGEs (default: BUNDLE_DEFAULT) into static libraries for +//! TRIPLE, for diffy-syntax's `bundled` feature (DIFFY_SYNTAX_BUNDLE=DIR). +//! Each is first built and checked as a host pack. Writes //: +//! lib/, queries//, bundle.tsv, THIRD_PARTY_NOTICES.md. +//! --out defaults to /target/syntax-bundle. //! syntax-pack registry --registry FILE [--extensions FILE] [--jobs N] //! Pin every grammar in nvim-treesitter's registry.json that is not in //! grammars.toml, build and check each for this host, and rewrite @@ -75,8 +74,8 @@ const HOST: &str = env!("SYNTAX_PACK_HOST"); const KEY_VAR: &str = "DIFFY_SYNTAX_SIGNING_KEY"; const TREE_SITTER_CLI: &str = "0.26.8"; const TREE_SITTER_VAR: &str = "TREE_SITTER"; -/// What `bundle` compiles in when no languages are named. `markdown_inline` -/// is there because markdown's injection query names it. +/// `bundle`'s languages when none are named; markdown injects +/// `markdown_inline`. const BUNDLE_DEFAULT: &[&str] = &[ "c", "cpp", @@ -458,8 +457,8 @@ fn inputs( Ok(inputs) } -/// Builds `language`'s pack; returns the directory of the C sources it -/// compiled (generated ones for a `generate` grammar). +/// Builds `language`'s pack; returns its C source directory (generated for +/// `generate` grammars). fn build(lock: &Lock, language: &str, options: &Options) -> Result { let grammar = lock .get(language) @@ -656,7 +655,6 @@ fn copy_dir(from: &Path, to: &Path) -> Result<(), String> { Ok(()) } -/// A C build of the sources in `src` for `target`, objects in `objects`. fn c_build(src: &Path, target: &str, objects: &Path) -> cc::Build { let mut build = cc::Build::new(); build @@ -724,8 +722,7 @@ fn compile( run_link(link, output) } -/// The command that links a shared library `output` for `target`; the -/// caller adds the inputs. +/// Linker command for the shared library `output`; the caller adds inputs. fn shared_link( compiler: &cc::Tool, target: &str, @@ -771,8 +768,8 @@ fn bundle_library(language: &str) -> String { format!("diffy_grammar_{name}") } -/// The file name `cc` gives the static library `name` for `target`, and -/// rustc looks for. +/// File name of static library `name` on `target`, as `cc` writes and +/// rustc expects. fn static_library_file(target: &str, name: &str) -> String { if target.contains("msvc") { format!("{name}.lib") @@ -800,9 +797,8 @@ fn bundle(lock: &Lock, options: &Options) -> Result<(), String> { let _ = fs::remove_dir_all(&out); let lib_dir = out.join("lib"); fs::create_dir_all(&lib_dir).map_err(|e| format!("{}: {e}", lib_dir.display()))?; - // Packs for this host, whatever the target: building one adapts its - // queries to tree-sitter and checks them, and its sample, against the - // grammar the bundle compiles from the same sources. + // Host packs, whatever the target: building one adapts its queries, and + // checking it tests them against the sources the bundle compiles. let host = Options { target: HOST.to_owned(), out: options.work.join("bundle-packs"), @@ -887,8 +883,8 @@ fn bundle(lock: &Lock, options: &Options) -> Result<(), String> { Ok(()) } -/// Compiles parser.c and its C scanner in `src` into the static library -/// `library` for `target`, copied into `lib_dir`. +/// Compiles `src`'s parser.c and C scanner into the static `library` in +/// `lib_dir`. fn compile_static( src: &Path, scanner: Scanner, @@ -900,8 +896,8 @@ fn compile_static( let _ = fs::remove_dir_all(objects); fs::create_dir_all(objects).map_err(|e| format!("{}: {e}", objects.display()))?; let mut build = c_build(src, target, objects); - // Position-independent so a shared library (a Node addon) can link it; - // the dynamic CRT on MSVC, as Rust uses by default. + // PIC so shared libraries (Node addons) can link it; dynamic CRT to + // match Rust on MSVC. build.pic(true).static_crt(false).file(src.join("parser.c")); if scanner == Scanner::C { build.file(src.join("scanner.c")); @@ -916,9 +912,8 @@ fn compile_static( Ok(()) } -/// Links every library of the bundle into one shared library, as a -/// consumer's build would link them into one binary: two grammars defining -/// the same global symbol fail here, for this target, instead of there. +/// Links the whole bundle into one shared library, so clashing global +/// symbols fail here rather than in a consumer's build. fn probe_link(options: &Options, symbols: &BTreeSet) -> Result<(), String> { let target = &options.target; let objects = options @@ -958,8 +953,8 @@ fn probe_link(options: &Options, symbols: &BTreeSet) -> Result<(), Strin run_link(link, &output).map_err(|e| format!("the bundle does not link as one: {e}")) } -/// Each bundled grammar's license, the repositories and commits its -/// sources and queries come from, and the license files found in them. +/// Per grammar: license, source repositories and commits, and their +/// license files. fn third_party_notices(lock: &Lock, options: &Options) -> Result { let mut text = String::from( "# Third-party notices\n\n\ @@ -978,16 +973,15 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result for (index, (repo, rev, root)) in sources.into_iter().enumerate() { text.push_str(&format!("Source: {repo} at {rev}\n\n")); let licenses = license_files(&root)?; - // The grammar's repository holds the C code compiled in; query - // repositories (nvim-treesitter-queries-*) carry no license file. + // The grammar repository's C code is compiled in, so it must + // ship a license. if licenses.is_empty() && index == 0 { return Err(format!( "{language}: {repo} at {rev} has no license file to ship" )); } if licenses.is_empty() && repo.contains("nvim-treesitter-queries-") { - // These repositories split out nvim-treesitter's queries, - // which it licenses Apache-2.0; the text follows the list. + // Split from nvim-treesitter, whose queries are Apache-2.0. nvim_queries = true; text.push_str( "Queries from nvim-treesitter, licensed Apache-2.0 \ @@ -1017,12 +1011,11 @@ fn third_party_notices(lock: &Lock, options: &Options) -> Result Ok(text) } -/// The Apache License 2.0, verbatim. const APACHE_2_0: &str = include_str!("../APACHE-2.0.txt"); -/// The repository, commit, and checkout of every source a language's -/// build reads: its grammar, its query repository, and other entries' -/// files it names (`@/`, `@:q/`). +/// Repository, commit, and checkout of every source `language`'s build +/// reads: its grammar, its queries, and other entries it names +/// (`@/`, `@:q/`). fn source_repositories( lock: &Lock, language: &str, @@ -1079,7 +1072,7 @@ fn source_repositories( Ok(found) } -/// The license files at the top of a checkout, by name. +/// License files at a checkout's top level, sorted. fn license_files(root: &Path) -> Result, String> { let entries = fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))?; let mut found: Vec = entries From 1fd0a5fa58d96c3b57f3e63bedfb3e70d91f558d Mon Sep 17 00:00:00 2001 From: rohit Date: Tue, 3 Nov 2026 06:55:39 +0000 Subject: [PATCH 16/16] ci(node): skip macOS dev codesigning and install libclang for tree-sitter-cli --- .github/workflows/node.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 3c749eee..8395c904 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -31,6 +31,8 @@ env: MACOSX_DEPLOYMENT_TARGET: "11.0" # The oldest glibc the Linux binaries may need (manylinux_2_28). MAX_GLIBC: "2.28" + # .cargo/config.toml's macOS runner signs with a local identity CI lacks. + DIFFY_SKIP_CODESIGN: "1" jobs: build: @@ -72,11 +74,12 @@ jobs: steps: # Before checkout, which needs git for the submodule. The image's GCC # replaces .cargo/config.toml's clang (keeping -fuse-ld=lld) so the - # grammars and the Rust link use the image's glibc. + # grammars and the Rust link use the image's glibc. libclang is for + # bindgen in tree-sitter-cli's QuickJS. - name: Install build tools (manylinux) if: matrix.container run: | - dnf install -y git lld + dnf install -y git lld clang-devel echo "CC=gcc" >> "$GITHUB_ENV" echo "CARGO_TARGET_$(echo "$TARGET" | tr 'a-z-' 'A-Z_')_LINKER=gcc" >> "$GITHUB_ENV"