From 4ab6b4f3a3f525c1529c5b853a05a6453cd88497 Mon Sep 17 00:00:00 2001 From: sunerpy Date: Tue, 14 Jul 2026 15:02:01 +0800 Subject: [PATCH] feat(config): add `include` to force gitignored first-party source into the index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports upstream colby #1063: an `[indexing] include` list of gitignore-style, root-relative path patterns (a dir, a `**` glob, or a single file) that forces first-party source INTO the index even when `.gitignore` excludes it — the case of a repo tracked by a second VCS (SVN/Perforce) whose sources are gitignored so they never land in git. Complements the existing `exclude` (its opposite). Off by default: an empty `include` leaves scan and watch behavior byte-identical. `include` is a separate decision applied AFTER the ordered gitignore/exclude model: a path the model would ignore is force-included iff it matches `include`, is not under a built-in skip (node_modules/.git/.godot/dist/…, never re-includable), and does not match an explicit config `exclude` (exclude wins). Precedence: built-in skip > exclude > include > the existing ordered ignore_paths/exclude/.gitignore model with `!`-negation, which is unchanged. The scanner also descends an otherwise-ignored directory when it is an ancestor of an include pattern, so an included file or nested glob under a gitignored parent is reached, while non-included files under it stay pruned. The force-include/exclude path decision is unified across crates via a single exported `codegraph_extract::include_exclude_pattern_matches` (whole-relative-path semantics), used by both the extractor scan and the watcher's WatchPolicy, so `codegraph index`, `codegraph sync`, and the live file-watcher agree on scope — `sync` output stays byte-for-byte equal to `index --force`. Threaded through ExtractOptions, WatchPolicy, WatchOptions, and DaemonOptions. Golden .schema and all fixtures unchanged (file-selection only; no regen). --- crates/codegraph-cli/src/main.rs | 7 + crates/codegraph-core/src/config.rs | 38 +++ crates/codegraph-daemon/src/lib.rs | 9 + crates/codegraph-extract/src/engine.rs | 277 +++++++++++++++++- crates/codegraph-extract/src/lib.rs | 5 +- crates/codegraph-watch/src/policy.rs | 151 +++++++++- crates/codegraph-watch/src/sync.rs | 124 +++++++- crates/codegraph-watch/src/watcher.rs | 17 +- .../tests/include_scan_watch_parity.rs | 111 +++++++ 9 files changed, 724 insertions(+), 15 deletions(-) create mode 100644 crates/codegraph-watch/tests/include_scan_watch_parity.rs diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index daeb3aec..b64a43ff 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -1424,11 +1424,14 @@ fn cmd_serve( return serve_direct(project, &project_root, no_watch, explicit_path); } ServeMode::BeDaemon => { + let cfg = codegraph_core::config::get_config(); return codegraph_daemon::run_foreground( &project_root, codegraph_daemon::DaemonOptions { run_mcp: true, host_pid: codegraph_daemon::host_pid_from_env(), + include: cfg.indexing.include.clone(), + exclude: cfg.indexing.exclude.clone(), ..Default::default() }, ) @@ -2096,6 +2099,9 @@ fn start_direct_watcher( ) -> Option { let mut opts = codegraph_watch::WatchOptions::default(); opts.no_watch = no_watch; + let cfg = codegraph_core::config::get_config(); + opts.include = cfg.indexing.include.clone(); + opts.exclude = cfg.indexing.exclude.clone(); opts.on_sync_complete = Some(std::sync::Arc::new( |outcome: codegraph_watch::SyncOutcome| { tracing::info!( @@ -3244,6 +3250,7 @@ fn index_project_inner( ignore_dirs: config.indexing.ignore_dirs.clone(), ignore_paths: config.indexing.ignore_paths.clone(), exclude: config.indexing.exclude.clone(), + include: config.indexing.include.clone(), parallel: true, }; if !quiet { diff --git a/crates/codegraph-core/src/config.rs b/crates/codegraph-core/src/config.rs index 9d8c2bbf..6d368fea 100644 --- a/crates/codegraph-core/src/config.rs +++ b/crates/codegraph-core/src/config.rs @@ -67,6 +67,18 @@ pub struct IndexingConfig { /// `docs/gen`, `gen*`); honored by index and sync. Off by default. #[serde(default)] pub exclude: Vec, + /// Root-relative path patterns for first-party source to force INTO the + /// index even when `.gitignore` (or the default `ignore_paths`) would drop + /// it — the case being a project under a second VCS (SVN, Perforce) that + /// `.gitignore`s its own real source out of Git yet still wants it indexed. + /// Same matcher as `.gitignore`/`exclude` (`Tools/`, `Local/ts/`, `gen*`), + /// matched against root-relative paths. Complements `exclude` (its + /// opposite). Precedence: a built-in `ignore_dirs` skip + /// (`node_modules`/`dist`/…) is NEVER re-included, and an explicit + /// `exclude` still wins over `include`. Off by default (empty = today's + /// behavior byte-identical). #1063. + #[serde(default)] + pub include: Vec, } fn default_max_file_size() -> u64 { @@ -187,6 +199,7 @@ impl Default for IndexingConfig { ignore_dirs: default_ignore_dirs(), ignore_paths: default_ignore_paths(), exclude: Vec::new(), + include: Vec::new(), } } } @@ -357,6 +370,31 @@ max_file_size = 2097152 assert!(cfg.indexing.exclude.is_empty()); } + #[test] + fn test_include_parses_and_defaults_empty() { + let with_include = r#" +[app] +name = "p" + +[indexing] +include = ["Tools/", "Local/ts/"] +"#; + let cfg: Config = toml::from_str(with_include).expect("should parse"); + assert_eq!(cfg.indexing.include, vec!["Tools/", "Local/ts/"]); + + let without = r#" +[app] +name = "p" + +[indexing] +max_file_size = 2097152 +"#; + let cfg: Config = toml::from_str(without).expect("should parse"); + assert!(cfg.indexing.include.is_empty()); + + assert!(IndexingConfig::default().include.is_empty()); + } + #[test] fn test_missing_file_returns_defaults() { let cfg = Config::discover(None, Path::new("/tmp/nonexistent")) diff --git a/crates/codegraph-daemon/src/lib.rs b/crates/codegraph-daemon/src/lib.rs index 84b5a353..a53ca5af 100644 --- a/crates/codegraph-daemon/src/lib.rs +++ b/crates/codegraph-daemon/src/lib.rs @@ -166,6 +166,11 @@ pub struct DaemonOptions { /// project (N client inotify sets collapse to 1). Honors /// `watch_disabled_reason` (e.g. `CODEGRAPH_NO_WATCH=1`). pub watch: bool, + /// `config.toml` `include`/`exclude` path patterns (#1063), passed through + /// to the shared `ProjectWatcher` so its scope matches the scan's. The CLI + /// (which owns the config singleton) populates these; empty = pre-#1063. + pub include: Vec, + pub exclude: Vec, } impl Default for DaemonOptions { @@ -176,6 +181,8 @@ impl Default for DaemonOptions { watchdog_interval: DEFAULT_WATCHDOG_INTERVAL, run_mcp: true, watch: true, + include: Vec::new(), + exclude: Vec::new(), } } } @@ -555,6 +562,8 @@ fn start_project_watcher( let counter = Arc::new(AtomicUsize::new(0)); let mut watch_options = codegraph_watch::WatchOptions::default(); watch_options.no_watch = !options.watch; + watch_options.include = options.include.clone(); + watch_options.exclude = options.exclude.clone(); watch_options.on_sync_complete = Some(Arc::new(move |outcome: codegraph_watch::SyncOutcome| { let n = counter.fetch_add(1, Ordering::SeqCst) + 1; diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index 486355de..eefafd17 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -24,6 +24,7 @@ pub struct ExtractOptions { pub ignore_dirs: Vec, pub ignore_paths: Vec, pub exclude: Vec, + pub include: Vec, pub parallel: bool, } @@ -35,6 +36,7 @@ impl Default for ExtractOptions { ignore_dirs: indexing.ignore_dirs, ignore_paths: indexing.ignore_paths, exclude: indexing.exclude, + include: indexing.include, parallel: true, } } @@ -233,7 +235,15 @@ pub fn scan_project(root: &Path, options: &ExtractOptions) -> Result // Evaluated in order (default paths → config exclude → .gitignore), so a // later `!pattern` negation re-includes a path an earlier set excluded. let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude, &gitignore]; - scan_dir(root, root, &ignored_dirs, &pattern_sets, &mut files)?; + let include = IncludeSet::new(&options.include, &options.exclude); + scan_dir( + root, + root, + &ignored_dirs, + &pattern_sets, + &include, + &mut files, + )?; files.sort(); Ok(files) } @@ -243,6 +253,7 @@ fn scan_dir( dir: &Path, ignored_dirs: &HashSet<&str>, pattern_sets: &[&[String]], + include: &IncludeSet, files: &mut Vec, ) -> Result<()> { let entries = fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))?; @@ -255,14 +266,26 @@ fn scan_dir( continue; } let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); - if is_path_ignored(&relative, pattern_sets) { - continue; - } + let ignored = is_path_ignored(&relative, pattern_sets); let file_type = entry.file_type()?; if file_type.is_dir() { - scan_dir(root, &path, ignored_dirs, pattern_sets, files)?; + // A model-ignored dir is normally pruned before descent, so a FILE + // include under a gitignored ancestor would never be reached. + // Descend anyway when this dir is an ancestor of (or matches) an + // include pattern; files inside are still pruned unless force-included. + if ignored && !include.wants_descend(&relative) { + continue; + } + scan_dir(root, &path, ignored_dirs, pattern_sets, include, files)?; } else if file_type.is_file() && is_extractable_source_path(&relative) { - files.push(relative); + // Post-model include decision: a model-ignored file is force-included + // iff it matches `include` and is NOT overridden by an explicit + // `exclude` (checked inside `IncludeSet::forces`). Built-in dir skips + // are already handled structurally above, so include can never + // resurface node_modules/dist/.git/etc. + if !ignored || include.forces(&relative) { + files.push(relative); + } } } Ok(()) @@ -341,6 +364,117 @@ fn pattern_matches(relative: &str, pattern: &str) -> bool { } } +/// Single source of truth for the `include`/`exclude` PATH-MATCH decision, +/// shared with `codegraph-watch` so the live watcher's scope is byte-identical +/// to the scan's (AGENTS.md "sync == index --force"). This is the WHOLE-relative +/// path `.gitignore`-style semantics of [`pattern_matches`] — NOT the watcher's +/// basename-glob `rule_matches`, which the two crates previously diverged on +/// (`gen*` matched `gen/helper.ts` in the scan but not the watcher). Argument +/// order is `(pattern, relative)` to read like "does this pattern match?". +pub fn include_exclude_pattern_matches(pattern: &str, relative: &str) -> bool { + pattern_matches(relative, pattern) +} + +/// The `include` force-inclusion decision (#1063), kept separate from the +/// ordered `.gitignore`-style model so it can flip a model-ignored path back in +/// AFTER that model returns its verdict. An explicit config `exclude` is checked +/// here so `exclude` always wins over `include`. A built-in `ignore_dirs` skip is +/// NOT re-checked — those are pruned structurally in `scan_dir` and can never +/// reach an include decision. Empty `include` makes every method a cheap `false`, +/// so the scan stays byte-identical to today. +struct IncludeSet<'a> { + include: &'a [String], + exclude: &'a [String], +} + +impl<'a> IncludeSet<'a> { + fn new(include: &'a [String], exclude: &'a [String]) -> Self { + Self { include, exclude } + } + + fn is_empty(&self) -> bool { + self.include.is_empty() + } + + /// A model-ignored FILE is force-included iff it matches an `include` + /// pattern and is not knocked out by an explicit `exclude` (exclude wins). + fn forces(&self, relative: &str) -> bool { + if self.is_empty() { + return false; + } + self.include + .iter() + .any(|p| include_file_matches(relative, p)) + && !self.exclude.iter().any(|p| pattern_matches(relative, p)) + } + + /// Whether a model-ignored DIRECTORY must still be descended: it either + /// matches an include pattern itself, or is an ANCESTOR of one (a nested + /// `Tools/gen/x.ts` include needs `Tools/` and `Tools/gen/` walked). An + /// explicit `exclude` on the directory prunes the whole subtree (exclude + /// wins), mirroring `forces`. + fn wants_descend(&self, relative: &str) -> bool { + if self.is_empty() { + return false; + } + if self.exclude.iter().any(|p| pattern_matches(relative, p)) { + return false; + } + self.include + .iter() + .any(|p| include_touches_dir(relative, p)) + } +} + +/// True when include `pattern` matches, or could match, something at or below +/// the directory `dir` (root-relative, no trailing slash) — i.e. whether the +/// ancestor dir of an included path must be descended. A bare name with no `/` +/// (e.g. `local`) names a file that `pattern_matches` accepts at ANY depth, so +/// like upstream's whole-tree walk it touches every dir. Otherwise the pattern +/// has a static path stem (dropping a trailing `/` or `*`): the dir is touched +/// when it is at/under the stem (`Tools` under `Tools/**`) or the stem is +/// at/under the dir (`Tools/gen` under the ancestor `Tools`). +fn include_touches_dir(dir: &str, pattern: &str) -> bool { + if !pattern.contains('/') && !pattern.ends_with('*') { + return true; + } + let stem = include_static_stem(pattern); + if stem.is_empty() { + return true; + } + dir == stem || dir.starts_with(&format!("{stem}/")) || stem.starts_with(&format!("{dir}/")) +} + +/// The literal leading directory of an include `pattern`, trailing slash / `*` / +/// `**` dropped: `Tools/` → `Tools`, `Tools/**` → `Tools`, `Local/ts/x.ts` → +/// `Local/ts/x.ts` (no glob, so the whole thing is literal). Used only for the +/// directory-descent ancestor test. +fn include_static_stem(pattern: &str) -> &str { + let p = pattern.trim_end_matches('/'); + match p.split_once('*') { + Some((prefix, _)) => prefix.trim_end_matches('/'), + None => p, + } +} + +/// Whether an include `pattern` force-includes the FILE at root-relative +/// `relative`. Extends `pattern_matches` with `**` support (`Tools/**` matches +/// every file under `Tools/`), which the ordered-model matcher deliberately does +/// not handle. A `dir/`-suffixed pattern matches any file under that dir; a +/// trailing `/**` (or bare `**`) matches everything under its prefix; the +/// remaining forms defer to `pattern_matches`. +fn include_file_matches(relative: &str, pattern: &str) -> bool { + if let Some(prefix) = pattern + .strip_suffix("/**") + .or_else(|| if pattern == "**" { Some("") } else { None }) + { + return prefix.is_empty() + || relative == prefix + || relative.starts_with(&format!("{prefix}/")); + } + pattern_matches(relative, pattern) +} + fn is_extractable_source_path(relative: &str) -> bool { let language = detect_language(relative); language != Language::Unknown @@ -483,6 +617,137 @@ mod tests { fs::remove_dir_all(&project).ok(); } + /// #1063 (a): a `.gitignore`d dir named in `include` is force-indexed. + #[test] + fn scan_include_forces_gitignored_dir_into_index() { + let project = unique_project("include_dir"); + touch(&project, ".gitignore", "Tools/\n"); + touch(&project, "src/app.ts", "export const a = 1;"); + touch(&project, "Tools/helper.ts", "export const b = 2;"); + + let options = ExtractOptions { + include: vec!["Tools/".to_string()], + ..ExtractOptions::default() + }; + let files = scan_project(&project, &options).expect("scan project"); + assert!( + files.contains(&"Tools/helper.ts".to_string()), + "gitignored Tools/ in include must be indexed: {files:?}" + ); + assert!(files.contains(&"src/app.ts".to_string())); + fs::remove_dir_all(&project).ok(); + } + + /// #1063 (b): a built-in skip (`node_modules`) named in `include` stays + /// skipped — include can never resurface a built-in ignored dir. + #[test] + fn scan_include_never_reincludes_builtin_skip() { + let project = unique_project("include_builtin"); + touch(&project, "src/app.ts", "export const a = 1;"); + touch(&project, "node_modules/pkg/index.ts", "export const x = 1;"); + + let options = ExtractOptions { + include: vec!["node_modules/".to_string()], + ..ExtractOptions::default() + }; + let files = scan_project(&project, &options).expect("scan project"); + assert!( + !files.iter().any(|f| f.starts_with("node_modules/")), + "include must not resurface a built-in skip: {files:?}" + ); + assert!(files.contains(&"src/app.ts".to_string())); + fs::remove_dir_all(&project).ok(); + } + + /// #1063 (c): a path in BOTH `include` and `exclude` is skipped — an + /// explicit `exclude` always wins over `include`. + #[test] + fn scan_exclude_wins_over_include() { + let project = unique_project("include_exclude"); + touch(&project, ".gitignore", "Tools/\n"); + touch(&project, "Tools/helper.ts", "export const b = 2;"); + + let options = ExtractOptions { + include: vec!["Tools/".to_string()], + exclude: vec!["Tools/".to_string()], + ..ExtractOptions::default() + }; + let files = scan_project(&project, &options).expect("scan project"); + assert!( + !files.iter().any(|f| f.starts_with("Tools/")), + "exclude must win over include: {files:?}" + ); + fs::remove_dir_all(&project).ok(); + } + + /// #1063 (d): an `include` naming a SINGLE FILE (or nested `**` glob) under a + /// gitignored ANCESTOR is indexed — the ancestor dir is descended even though + /// the ordered model would prune it, while non-included siblings stay pruned. + #[test] + fn scan_include_reaches_file_under_gitignored_ancestor() { + let project = unique_project("include_ancestor"); + touch(&project, ".gitignore", "Local/\n"); + touch(&project, "Local/ts/wanted.ts", "export const w = 1;"); + touch(&project, "Local/ts/other.ts", "export const o = 2;"); + touch(&project, "Local/skip.ts", "export const s = 3;"); + + let options = ExtractOptions { + include: vec!["Local/ts/wanted.ts".to_string()], + ..ExtractOptions::default() + }; + let files = scan_project(&project, &options).expect("scan project"); + assert!( + files.contains(&"Local/ts/wanted.ts".to_string()), + "a file include under a gitignored ancestor must be indexed: {files:?}" + ); + assert!( + !files.contains(&"Local/ts/other.ts".to_string()) + && !files.contains(&"Local/skip.ts".to_string()), + "non-included siblings under the ancestor stay pruned: {files:?}" + ); + + let glob = ExtractOptions { + include: vec!["Local/ts/**".to_string()], + ..ExtractOptions::default() + }; + let files = scan_project(&project, &glob).expect("scan project"); + assert!( + files.contains(&"Local/ts/wanted.ts".to_string()) + && files.contains(&"Local/ts/other.ts".to_string()), + "a nested ** include pulls in the whole subdir: {files:?}" + ); + assert!( + !files.contains(&"Local/skip.ts".to_string()), + "the ** glob does not reach a sibling outside its dir: {files:?}" + ); + fs::remove_dir_all(&project).ok(); + } + + /// #1063 (e): empty `include` leaves the scanned file set byte-identical. + #[test] + fn scan_empty_include_is_byte_identical() { + let project = unique_project("include_empty"); + touch(&project, ".gitignore", "vendor/\n"); + touch(&project, "src/app.ts", "export const a = 1;"); + touch(&project, "vendor/dep.ts", "export const d = 2;"); + + let base = scan_project(&project, &ExtractOptions::default()).expect("scan"); + let with_empty = scan_project( + &project, + &ExtractOptions { + include: Vec::new(), + ..ExtractOptions::default() + }, + ) + .expect("scan"); + assert_eq!( + base, with_empty, + "empty include must not change the file set" + ); + assert!(!base.iter().any(|f| f.starts_with("vendor/"))); + fs::remove_dir_all(&project).ok(); + } + #[test] fn extract_file_reads_and_parses_a_real_source_file() { let project = unique_project("extract_file"); diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index 504cf3c1..5fb99827 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -16,4 +16,7 @@ pub mod lang; pub mod spec; pub mod walker; -pub use engine::{ExtractOptions, detect_language, extract_file, extract_project, extract_source}; +pub use engine::{ + ExtractOptions, detect_language, extract_file, extract_project, extract_source, + include_exclude_pattern_matches, +}; diff --git a/crates/codegraph-watch/src/policy.rs b/crates/codegraph-watch/src/policy.rs index c74b316a..03f72114 100644 --- a/crates/codegraph-watch/src/policy.rs +++ b/crates/codegraph-watch/src/policy.rs @@ -80,10 +80,25 @@ struct IgnoreRule { pub struct WatchPolicy { root: PathBuf, rules: Vec, + /// Number of leading `rules` that are the built-in skip set (never + /// re-includable by `include`); the rest are `.gitignore`-derived. + builtin_rule_count: usize, + include: Vec, + exclude: Vec, } impl WatchPolicy { pub fn new(root: impl AsRef) -> Self { + Self::with_config(root, &[], &[]) + } + + /// Like [`new`](Self::new) but threads the `codegraph.json`/`config.toml` + /// `include` and `exclude` path patterns (#1063) so the watcher's scope + /// matches the scan's: an included gitignored dir is WATCHED and its file + /// events HANDLED, while a built-in skip is never re-watched and an explicit + /// `exclude` always wins. Empty `include`+`exclude` is byte-identical to the + /// pre-#1063 policy. + pub fn with_config(root: impl AsRef, include: &[String], exclude: &[String]) -> Self { // Mirrors the upstream built-in ignore seed and root .gitignore merge from // `upstream extraction/index.ts:117-161,242-246` so // `.gitignore` negations can opt default-excluded dirs back in. @@ -109,8 +124,19 @@ impl WatchPolicy { negated: false, }, ]); + // The built-in DEFAULT_IGNORE_DIRS + egg/cmake/bazel rules are the + // "built-in skip" set that `include` must NEVER re-watch; count them so + // the include override below can tell a built-in skip from a `.gitignore` + // rule. Only `.gitignore` (and default-path) exclusions are overridable. + let builtin_rule_count = rules.len(); rules.extend(read_gitignore_rules(&root)); - Self { root, rules } + Self { + root, + rules, + builtin_rule_count, + include: include.to_vec(), + exclude: exclude.to_vec(), + } } pub fn normalize_relative(&self, path: impl AsRef) -> Option { @@ -149,8 +175,41 @@ impl WatchPolicy { } fn is_ignored(&self, relative: &str, is_dir: bool) -> bool { + let base = self.matches_rules(&self.rules, relative, is_dir); + // Empty include short-circuits the #1063 override → pre-#1063 behavior + // byte-identical (the override only ever flips an ignored path back in). + if self.include.is_empty() || !base { + return base; + } + // base == ignored: consider the include force-inclusion override. + // Precedence: a built-in skip is NEVER re-watched; an explicit `exclude` + // wins over `include`; otherwise an included dir (or an ancestor of an + // included path) is watched and an included file is handled. + if self.matches_rules(&self.rules[..self.builtin_rule_count], relative, is_dir) { + return true; + } + if self + .exclude + .iter() + .any(|pattern| codegraph_extract::include_exclude_pattern_matches(pattern, relative)) + { + return true; + } + let force = if is_dir { + self.include + .iter() + .any(|pattern| include_touches_dir(relative, pattern)) + } else { + self.include + .iter() + .any(|pattern| include_file_matches(relative, pattern)) + }; + !force + } + + fn matches_rules(&self, rules: &[IgnoreRule], relative: &str, is_dir: bool) -> bool { let mut ignored = false; - for rule in &self.rules { + for rule in rules { if rule_matches(&rule.pattern, relative, is_dir) { ignored = !rule.negated; } @@ -159,6 +218,47 @@ impl WatchPolicy { } } +/// Whether an include `pattern` matches the FILE at root-relative `relative` +/// (watcher side, byte-identical to the engine's `include_file_matches`): a +/// `dir/**` (or bare `**`) matches every file under its prefix, and every other +/// form defers to the SHARED whole-path matcher `include_exclude_pattern_matches` +/// — NOT the watcher's basename-glob `rule_matches` — so `gen*` matches +/// `gen/helper.ts` here exactly as it does in the scan. +fn include_file_matches(relative: &str, pattern: &str) -> bool { + if let Some(prefix) = pattern + .strip_suffix("/**") + .or_else(|| (pattern == "**").then_some("")) + { + return prefix.is_empty() + || relative == prefix + || relative.starts_with(&format!("{prefix}/")); + } + codegraph_extract::include_exclude_pattern_matches(pattern, relative) +} + +/// Whether an include `pattern` matches, or is an ancestor of, the DIRECTORY +/// `relative` — so a gitignored ancestor of an included path is still watched +/// (watcher side, mirroring the engine's `include_touches_dir`). A bare-name +/// pattern matches at any depth, so it touches every dir. +fn include_touches_dir(relative: &str, pattern: &str) -> bool { + if !pattern.contains('/') && !pattern.ends_with('*') { + return true; + } + let stem = { + let trimmed = pattern.trim_end_matches('/'); + match trimmed.split_once('*') { + Some((prefix, _)) => prefix.trim_end_matches('/'), + None => trimmed, + } + }; + if stem.is_empty() { + return true; + } + relative == stem + || relative.starts_with(&format!("{stem}/")) + || stem.starts_with(&format!("{relative}/")) +} + pub fn watch_disabled_reason(project_root: impl AsRef, no_watch: bool) -> Option { // Port of `upstream sync/watch-policy.ts:77-95`: explicit opt-out // wins, force-watch overrides auto detection, WSL /mnt drives are disabled. @@ -666,6 +766,53 @@ mod tests { assert_eq!(normalize_path("a/b/c"), "a/b/c"); } + #[test] + fn with_config_include_watches_gitignored_path_but_not_builtin_skip() { + // #1063: an included gitignored dir is watched and its files handled, + // while a built-in skip named in include stays pruned and an explicit + // exclude still wins. + let dir = crate::sync::tests::TestDir::new("watch-policy-include"); + fs::write(dir.path().join(".gitignore"), "Tools/\nLocal/\n").unwrap(); + let policy = WatchPolicy::with_config( + dir.path(), + &["Tools/".to_string(), "Local/ts/app.ts".to_string()], + &[], + ); + + // An included gitignored dir (and a file under it) is now watched/handled. + assert!(policy.should_watch_dir("Tools")); + assert!(policy.should_handle_file("Tools/helper.ts")); + // A file include under a gitignored ancestor: the ancestor is watched and + // the file handled, but a non-included sibling stays ignored. + assert!(policy.should_watch_dir("Local")); + assert!(policy.should_watch_dir("Local/ts")); + assert!(policy.should_handle_file("Local/ts/app.ts")); + assert!(!policy.should_handle_file("Local/ts/other.ts")); + // A built-in skip is NEVER re-watched, even if named in include. + let builtin = WatchPolicy::with_config(dir.path(), &["node_modules/".to_string()], &[]); + assert!(!builtin.should_watch_dir("node_modules")); + assert!(!builtin.should_handle_file("node_modules/pkg/index.ts")); + // An explicit exclude wins over include. + let excluded = + WatchPolicy::with_config(dir.path(), &["Tools/".to_string()], &["Tools/".to_string()]); + assert!(!excluded.should_watch_dir("Tools")); + assert!(!excluded.should_handle_file("Tools/helper.ts")); + } + + #[test] + fn empty_include_leaves_policy_byte_identical() { + // #1063: WatchPolicy::new == with_config(root, &[], &[]); an empty include + // must not change any watch/handle decision vs the pre-#1063 policy. + let dir = crate::sync::tests::TestDir::new("watch-policy-include-empty"); + fs::write(dir.path().join(".gitignore"), "vendor/\n").unwrap(); + let policy = WatchPolicy::with_config(dir.path(), &[], &[]); + assert!(!policy.should_watch_dir("vendor")); + assert!(!policy.should_handle_file("vendor/dep.ts")); + assert!(!policy.should_watch_dir("node_modules")); + assert!(policy.should_watch_dir("src")); + assert!(policy.should_handle_file("src/app.ts")); + } + #[test] fn should_handle_file_requires_known_language() { let dir = crate::sync::tests::TestDir::new("watch-policy-handle"); diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 633de105..87f19fee 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -44,6 +44,7 @@ pub fn sync_project_once_with_progress( ignore_dirs: config.indexing.ignore_dirs.clone(), ignore_paths: config.indexing.ignore_paths.clone(), exclude: config.indexing.exclude.clone(), + include: config.indexing.include.clone(), parallel: true, }; let started = std::time::Instant::now(); @@ -63,29 +64,62 @@ pub fn sync_project_once_with_progress( } } - sync_paths_with_store(&mut store, project_root, candidates, started, on_progress) + sync_paths_with_store( + &mut store, + project_root, + candidates, + &config.indexing.include, + &config.indexing.exclude, + started, + on_progress, + ) } pub fn sync_changed_paths( project_root: impl AsRef, db_path: impl AsRef, paths: impl IntoIterator>, +) -> Result { + sync_changed_paths_with_patterns(project_root, db_path, paths, &[], &[]) +} + +/// Like [`sync_changed_paths`] but threads the `codegraph.json`/`config.toml` +/// `include`/`exclude` path patterns (#1063) so the watcher's incremental sync +/// scopes files exactly as the full scan does — an included gitignored path is +/// handled, a built-in skip is not, and an explicit `exclude` wins. +pub fn sync_changed_paths_with_patterns( + project_root: impl AsRef, + db_path: impl AsRef, + paths: impl IntoIterator>, + include: &[String], + exclude: &[String], ) -> Result { let started = std::time::Instant::now(); let project_root = project_root.as_ref(); let db_path = db_path.as_ref(); let mut store = Store::open(db_path).with_context(|| format!("open {}", db_path.display()))?; - sync_paths_with_store(&mut store, project_root, paths, started, |_, _| {}) + sync_paths_with_store( + &mut store, + project_root, + paths, + include, + exclude, + started, + |_, _| {}, + ) } +#[allow(clippy::too_many_arguments)] fn sync_paths_with_store( store: &mut Store, project_root: &Path, paths: impl IntoIterator>, + include: &[String], + exclude: &[String], started: std::time::Instant, mut on_progress: impl FnMut(usize, usize), ) -> Result { - let policy = WatchPolicy::new(project_root); + let policy = WatchPolicy::with_config(project_root, include, exclude); let mut outcome = SyncOutcome::default(); let mut changed = false; let mut seen = HashSet::new(); @@ -909,6 +943,90 @@ pub(crate) mod tests { ); } + #[test] + fn sync_project_once_indexes_gitignored_dir_named_in_include() { + // #1063: a gitignored first-party dir named in `[indexing] include` is + // indexed on a full `codegraph sync` (sync_project_once reads the config + // and threads include into both the scan and the WatchPolicy gate). + let dir = TestDir::new("watch-once-include"); + fs::create_dir_all(dir.path().join(".codegraph")).unwrap(); + fs::write(dir.path().join(".gitignore"), "Tools/\n").unwrap(); + fs::write( + dir.path().join(".codegraph/config.toml"), + "[app]\nname = \"p\"\n\n[indexing]\ninclude = [\"Tools/\"]\n", + ) + .unwrap(); + fs::create_dir_all(dir.path().join("src")).unwrap(); + fs::create_dir_all(dir.path().join("Tools")).unwrap(); + fs::write( + dir.path().join("src/app.ts"), + "export function a() { return 1; }\n", + ) + .unwrap(); + fs::write( + dir.path().join("Tools/helper.ts"), + "export function help() { return 2; }\n", + ) + .unwrap(); + // The config OnceLock is process-global; point it at THIS project (a + // repeat set is an ignorable error, but then it may hold another test's + // path). Guard: only assert when our config actually took effect. + let cfg = codegraph_core::config::init_config(None, dir.path()); + if cfg + .map(|c| c.indexing.include.iter().any(|p| p == "Tools/")) + .unwrap_or(false) + { + let outcome = sync_project_once(dir.path()).unwrap(); + assert!( + outcome + .changed_paths + .contains(&"Tools/helper.ts".to_string()) + ); + let store = Store::open(&default_db_path(dir.path())).unwrap(); + assert!( + store.file_by_path("Tools/helper.ts").unwrap().is_some(), + "gitignored Tools/ named in include must be indexed on sync" + ); + } + } + + #[test] + fn sync_changed_paths_with_patterns_handles_included_gitignored_file() { + // #1063: the watcher's incremental path uses + // sync_changed_paths_with_patterns; a gitignored file named in include is + // reindexed rather than ignored by the WatchPolicy gate. + let dir = TestDir::new("watch-incr-include"); + fs::write(dir.path().join(".gitignore"), "Tools/\n").unwrap(); + fs::create_dir_all(dir.path().join("Tools")).unwrap(); + fs::write( + dir.path().join("Tools/helper.ts"), + "export function help() { return 1; }\n", + ) + .unwrap(); + let db = default_db_path(dir.path()); + + // Without include: the gitignored file is ignored by the policy gate. + let plain = sync_changed_paths(dir.path(), &db, ["Tools/helper.ts"]).unwrap(); + assert_eq!(plain.files_reindexed, 0); + assert!(plain.files_ignored >= 1); + + // With include: the same file is now handled and reindexed. + let included = sync_changed_paths_with_patterns( + dir.path(), + &db, + ["Tools/helper.ts"], + &["Tools/".to_string()], + &[], + ) + .unwrap(); + assert_eq!(included.files_reindexed, 1); + assert!( + included + .changed_paths + .contains(&"Tools/helper.ts".to_string()) + ); + } + #[test] fn deleting_an_imported_module_refreshes_its_dependents() { // Given: a helper module and a consumer that resolves an import to it, diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index 23d92982..264264de 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -12,7 +12,7 @@ use anyhow::Result; use notify::{Event, RecommendedWatcher, RecursiveMode, Watcher}; use crate::policy::{WatchPolicy, watch_disabled_reason}; -use crate::sync::{SyncOutcome, default_db_path, sync_changed_paths}; +use crate::sync::{SyncOutcome, default_db_path, sync_changed_paths_with_patterns}; type SyncCallback = Arc; type SyncFn = Arc) -> Result + Send + Sync>; @@ -215,6 +215,11 @@ pub struct WatchOptions { /// Called for a non-degrading watch/sync error (e.g. inotify watch-count /// exhaustion). May fire more than once; the watcher keeps running. pub on_sync_error: Option, + /// `codegraph.json`/`config.toml` `include`/`exclude` path patterns (#1063). + /// Threaded into the [`WatchPolicy`] and the default incremental-sync fn so + /// the watcher's scope matches the scan's. Empty = pre-#1063 behavior. + pub include: Vec, + pub exclude: Vec, sync_fn: Option, } @@ -230,6 +235,8 @@ impl Default for WatchOptions { on_sync_complete: None, on_degraded: None, on_sync_error: None, + include: Vec::new(), + exclude: Vec::new(), sync_fn: None, } } @@ -262,14 +269,18 @@ impl ProjectWatcher { if watch_disabled_reason(&project_root, options.no_watch).is_some() { return Ok(None); } - let policy = WatchPolicy::new(&project_root); + let policy = WatchPolicy::with_config(&project_root, &options.include, &options.exclude); let db_path = options .db_path .clone() .unwrap_or_else(|| default_db_path(&project_root)); let sync_fn = options.sync_fn.clone().unwrap_or_else(|| { let project_root = project_root.clone(); - Arc::new(move |paths| sync_changed_paths(&project_root, &db_path, paths)) + let include = options.include.clone(); + let exclude = options.exclude.clone(); + Arc::new(move |paths| { + sync_changed_paths_with_patterns(&project_root, &db_path, paths, &include, &exclude) + }) }); let (tx, rx) = mpsc::channel(); let degraded = Arc::new(DegradedState::default()); diff --git a/crates/codegraph-watch/tests/include_scan_watch_parity.rs b/crates/codegraph-watch/tests/include_scan_watch_parity.rs new file mode 100644 index 00000000..e6f55552 --- /dev/null +++ b/crates/codegraph-watch/tests/include_scan_watch_parity.rs @@ -0,0 +1,111 @@ +//! #1063 scan⇔watch parity: the `include`/`exclude` PATH-MATCH decision must be +//! byte-identical between the engine scan (`codegraph index`) and the live +//! watcher (`WatchPolicy`), or `sync`/watch would drop a file `index` kept — +//! violating the AGENTS.md "sync == index --force" invariant. This guards the +//! `gen*` whole-path-vs-basename divergence specifically. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::SystemTime; + +use codegraph_extract::{ExtractOptions, engine::scan_project}; +use codegraph_watch::WatchPolicy; + +fn unique_project(tag: &str) -> PathBuf { + static COUNTER: AtomicU32 = AtomicU32::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "cg_parity_{tag}_{}_{nanos}_{n}", + std::process::id() + )); + fs::create_dir_all(&dir).expect("create temp project"); + dir +} + +fn touch(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).expect("create parent dirs"); + fs::write(&path, contents).expect("write file"); +} + +/// For each include pattern form, a file's membership in the engine scan output +/// must equal `WatchPolicy::should_handle_file`, proving the two matchers agree. +#[test] +fn scan_and_watch_agree_on_include_file_verdicts() { + let project = unique_project("include"); + touch(&project, ".gitignore", "gen/\nTools/\nLocal/\n"); + touch(&project, "src/app.ts", "export const a = 1;"); + touch(&project, "gen/helper.ts", "export const g = 1;"); + touch(&project, "Tools/helper.ts", "export const t = 1;"); + touch(&project, "Local/ts/wanted.ts", "export const w = 1;"); + touch(&project, "Local/ts/other.ts", "export const o = 1;"); + touch(&project, "Local/skip.ts", "export const s = 1;"); + + let candidate_files = [ + "src/app.ts", + "gen/helper.ts", + "Tools/helper.ts", + "Local/ts/wanted.ts", + "Local/ts/other.ts", + "Local/skip.ts", + ]; + + for include in [ + vec!["gen*".to_string()], + vec!["Tools/".to_string()], + vec!["Local/ts/**".to_string()], + vec!["Local/ts/".to_string()], + ] { + let options = ExtractOptions { + include: include.clone(), + ..ExtractOptions::default() + }; + let scanned = scan_project(&project, &options).expect("scan"); + let policy = WatchPolicy::with_config(&project, &include, &[]); + + for file in candidate_files { + let in_scan = scanned.iter().any(|f| f == file); + let watched = policy.should_handle_file(file); + assert_eq!( + in_scan, watched, + "scan⇔watch parity broken for include={include:?} file={file}: \ + scan={in_scan} watch={watched}" + ); + } + } +} + +/// The specific #1063 blocker: `gen*` (a documented supported form) must both +/// INDEX and WATCH a gitignored `gen/helper.ts` consistently. +#[test] +fn gen_glob_indexes_and_watches_gitignored_file() { + let project = unique_project("gen_glob"); + touch(&project, ".gitignore", "gen/\n"); + touch(&project, "gen/helper.ts", "export const g = 1;"); + + let include = vec!["gen*".to_string()]; + let options = ExtractOptions { + include: include.clone(), + ..ExtractOptions::default() + }; + let scanned = scan_project(&project, &options).expect("scan"); + assert!( + scanned.iter().any(|f| f == "gen/helper.ts"), + "gen* must index gen/helper.ts: {scanned:?}" + ); + + let policy = WatchPolicy::with_config(&project, &include, &[]); + assert!( + policy.should_handle_file("gen/helper.ts"), + "gen* must watch gen/helper.ts (parity with scan)" + ); + assert!( + policy.should_watch_dir("gen"), + "gen* must keep the gen/ dir watchable" + ); +}