diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index af0e0d7d..daeb3aec 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -1141,8 +1141,20 @@ fn cmd_status(path: Option, json_output: bool) -> Result<()> { .and_then(|v| v.parse::().ok()); let reindex_recommended = last_indexed.is_some() && built_with_extraction_version.is_none_or(|v| v < EXTRACTION_VERSION); + let resolution_incomplete = store.is_resolution_incomplete()?; if json_output { + let mut index_obj = json!({ + "builtWithVersion": built_with_version, + "builtWithExtractionVersion": built_with_extraction_version, + "currentExtractionVersion": EXTRACTION_VERSION, + "reindexRecommended": reindex_recommended, + }); + // #1187: surface the interrupted-index state ONLY when the marker is set, + // so a healthy index's status JSON is byte-identical to a pre-#1187 build. + if resolution_incomplete { + index_obj["partial"] = json!(true); + } print_json(&json!({ "initialized": true, "version": VERSION, @@ -1159,12 +1171,7 @@ fn cmd_status(path: Option, json_output: bool) -> Result<()> { "languages": files_by_language.iter().filter(|(_, c)| *c > 0).map(|(l, _)| l).collect::>(), "pendingChanges": { "added": 0, "modified": 0, "removed": 0 }, "worktreeMismatch": null, - "index": { - "builtWithVersion": built_with_version, - "builtWithExtractionVersion": built_with_extraction_version, - "currentExtractionVersion": EXTRACTION_VERSION, - "reindexRecommended": reindex_recommended, - }, + "index": index_obj, "dbPath": db, "dbExists": db_exists, "daemonRunning": daemon_running, @@ -1197,7 +1204,13 @@ fn cmd_status(path: Option, json_output: bool) -> Result<()> { for (language, count) in files_by_language { println!(" {language:15} {}", format_number(count)); } - println!("\nIndex is up to date\n"); + if resolution_incomplete { + println!( + "\n⚠ Index is PARTIAL: a resolution pass was interrupted, so some call\n edges are missing. Run `codegraph sync` to heal it.\n" + ); + } else { + println!("\nIndex is up to date\n"); + } Ok(()) } @@ -1415,6 +1428,7 @@ fn cmd_serve( &project_root, codegraph_daemon::DaemonOptions { run_mcp: true, + host_pid: codegraph_daemon::host_pid_from_env(), ..Default::default() }, ) @@ -3492,17 +3506,29 @@ fn index_project_inner( drop(pending_nodes); finish_phase(&pb, "Persisted nodes"); + // WAL-valve fold threshold (#1231): with bulk autocheckpoint deferred + // (set_bulk_index_pragmas), fold the WAL back whenever it grows past this + // size so it never balloons unbounded across the edge/ref replay passes. + let wal_valve_bytes = codegraph_store::wal_valve_threshold_bytes(); let mut spill = spill.into_reader()?; let pb = phase_spinner("Persisting edges", quiet); spill.replay_edges(EDGE_FLUSH_ROWS, |batch| { - store.insert_edges(batch).map_err(anyhow::Error::from) + store.insert_edges(batch).map_err(anyhow::Error::from)?; + store + .checkpoint_wal_if_over(wal_valve_bytes) + .map_err(anyhow::Error::from)?; + Ok(()) })?; finish_phase(&pb, "Persisted edges"); let pb = phase_spinner("Persisting references", quiet); spill.replay_refs(REF_FLUSH_ROWS, |batch| { store .insert_unresolved_refs(batch) - .map_err(anyhow::Error::from) + .map_err(anyhow::Error::from)?; + store + .checkpoint_wal_if_over(wal_valve_bytes) + .map_err(anyhow::Error::from)?; + Ok(()) })?; finish_phase(&pb, "Persisted references"); spill.cleanup(); diff --git a/crates/codegraph-cli/tests/status_debug_fields.rs b/crates/codegraph-cli/tests/status_debug_fields.rs index 14979d49..3f71a9c8 100644 --- a/crates/codegraph-cli/tests/status_debug_fields.rs +++ b/crates/codegraph-cli/tests/status_debug_fields.rs @@ -142,4 +142,52 @@ fn status_json_exposes_debug_fields_initialized() { assert_eq!(v["initialized"], Value::Bool(true), "must be initialized"); assert_eq!(v["dbExists"], Value::Bool(true), "db must exist post-init"); assert_debug_fields(&v); + // #1187: a healthy index must NOT carry the partial flag. + assert!( + v["index"].get("partial").is_none(), + "a healthy index must omit index.partial: {v}" + ); +} + +#[test] +fn status_json_flags_partial_only_when_marker_set() { + use codegraph_store::Store; + + // GIVEN an indexed project with the #1187 incomplete-resolution marker set, + // simulating an interrupted resolution pass. + let dir = TestDir::new("partial"); + let project = dir.path().join("mini"); + copy_tree(&mini_fixture(), &project); + let p = project.to_str().unwrap(); + let (out, err, ok) = cli(&["init", p]); + assert!(ok, "init failed: stdout={out} stderr={err}"); + + let db = project.join(".codegraph").join("codegraph.db"); + { + let store = Store::open(&db).unwrap(); + store.set_resolution_incomplete().unwrap(); + } + + // WHEN `status --json` runs. + let (out, err, ok) = cli(&["status", "--json", p]); + assert!(ok, "status failed: stdout={out} stderr={err}"); + let v: Value = serde_json::from_str(out.trim()).expect("valid JSON"); + + // THEN index.partial is true. + assert_eq!( + v["index"]["partial"], + Value::Bool(true), + "index.partial must be true when the marker is set: {v}" + ); + + // AND after a heal sync the flag is gone again. + let (out, err, ok) = cli(&["sync", p]); + assert!(ok, "sync failed: stdout={out} stderr={err}"); + let (out, err, ok) = cli(&["status", "--json", p]); + assert!(ok, "status failed: stdout={out} stderr={err}"); + let v: Value = serde_json::from_str(out.trim()).expect("valid JSON"); + assert!( + v["index"].get("partial").is_none(), + "a healed index must drop index.partial: {v}" + ); } diff --git a/crates/codegraph-daemon/src/lib.rs b/crates/codegraph-daemon/src/lib.rs index c80fa137..84b5a353 100644 --- a/crates/codegraph-daemon/src/lib.rs +++ b/crates/codegraph-daemon/src/lib.rs @@ -66,6 +66,17 @@ pub const CODEGRAPH_DAEMON_MAX_IDLE_MS: &str = "CODEGRAPH_DAEMON_MAX_IDLE_MS"; /// colby `DEFAULT_CLIENT_SWEEP_MS`. pub const CODEGRAPH_DAEMON_CLIENT_SWEEP_MS: &str = "CODEGRAPH_DAEMON_CLIENT_SWEEP_MS"; +/// Env var name: the real MCP host pid, threaded down by the launcher so the +/// daemon watchdog polls the host directly (#1185). Unset/invalid/`0` → `None`, +/// which preserves the pre-#1185 behavior of watching only the launcher. +pub const CODEGRAPH_HOST_PPID: &str = "CODEGRAPH_HOST_PPID"; + +/// Env var name: milliseconds a freshly-accepting daemon waits for its FIRST +/// client request before assuming it was orphaned during startup and self-exits +/// (#1185). Default 900000 (15 min); `0` disables. Disarmed on the first +/// accepted connection, so a live session is never touched. +pub const CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS: &str = "CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS"; + const DEFAULT_IDLE_TIMEOUT_MS: u128 = 300_000; const DEFAULT_MAX_IDLE_MS: u128 = 1_800_000; const MIN_IDLE_TIMEOUT_MS: u128 = 1_000; @@ -75,6 +86,8 @@ const DEFAULT_CLIENT_SWEEP_MS: u128 = 30_000; const MIN_CLIENT_SWEEP_MS: u128 = 50; const MAX_CLIENT_SWEEP_MS: u128 = 600_000; +const DEFAULT_STARTUP_HANDSHAKE_TIMEOUT_MS: u128 = 900_000; + /// Parse + clamp the idle-timeout env var (mirror colby `resolveIdleTimeoutMs`): /// unset/empty/invalid -> default; otherwise clamp to [MIN, MAX]. fn resolve_idle_timeout_ms() -> u128 { @@ -114,6 +127,35 @@ fn resolve_client_sweep_ms() -> u128 { ) } +/// Read the real host pid from `CODEGRAPH_HOST_PPID` (#1185). Unset, empty, +/// non-numeric, or `0` → `None` (watch only the launcher, pre-#1185 behavior). +/// Exposed as [`host_pid_from_env`] so the CLI launch site can populate +/// `DaemonOptions.host_pid` without duplicating the parse. +fn resolve_host_pid_from_env() -> Option { + match std::env::var(CODEGRAPH_HOST_PPID) { + Ok(raw) if !raw.is_empty() => raw.parse::().ok().filter(|&pid| pid != 0), + _ => None, + } +} + +/// Public accessor for the real host pid parsed from `CODEGRAPH_HOST_PPID` +/// (#1185); see [`resolve_host_pid_from_env`]. +pub fn host_pid_from_env() -> Option { + resolve_host_pid_from_env() +} + +/// Startup-handshake timeout (#1185): unset/empty/invalid → default; `0` disables. +/// Unlike the idle timeouts this is NOT clamped to a floor — `0` must survive as +/// the explicit disable signal. +fn resolve_startup_handshake_timeout_ms() -> u128 { + match std::env::var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS) { + Ok(raw) if !raw.is_empty() => raw + .parse::() + .unwrap_or(DEFAULT_STARTUP_HANDSHAKE_TIMEOUT_MS), + _ => DEFAULT_STARTUP_HANDSHAKE_TIMEOUT_MS, + } +} + #[derive(Clone, Debug)] pub struct DaemonOptions { pub parent_pid: Option, @@ -377,6 +419,15 @@ async fn run_accept_loop_async( let idle_timeout_ms = resolve_idle_timeout_ms(); let max_idle_ms = resolve_max_idle_ms(); let client_sweep_ms = resolve_client_sweep_ms(); + // Startup-handshake backstop (#1185): a daemon whose launcher was killed + // mid-startup while its stdio pipes stayed open is orphaned — it received no + // client at all. Arm a one-shot timer from accept-loop entry and disarm it on + // the FIRST accepted connection; if it fires with zero connections the daemon + // self-exits. `0` disables. A live session that connects even once is never + // touched (the idle/max-idle/supervision paths own its later lifecycle). + let startup_handshake_ms = resolve_startup_handshake_timeout_ms(); + let accept_loop_started = std::time::Instant::now(); + let mut first_connection_seen = false; info!(project = %project_root.display(), socket = %socket_path.display(), "daemon started"); // ONE shared watcher per daemon process. Bound to a local so @@ -401,6 +452,7 @@ async fn run_accept_loop_async( tokio::select! { accepted = listener.accept() => match accepted { Ok(stream) => { + first_connection_seen = true; let session_project = project_root.clone(); let session_socket = socket_display.clone(); let session_registry = registry.clone(); @@ -434,14 +486,27 @@ async fn run_accept_loop_async( warn!(reason, "daemon watchdog stopping after supervisor loss"); break None; } + if startup_handshake_ms > 0 + && !first_connection_seen + && accept_loop_started.elapsed().as_millis() >= startup_handshake_ms + { + warn!( + startup_handshake_ms, + "daemon exiting on startup-handshake backstop: no client connected" + ); + break None; + } if last_sweep.elapsed().as_millis() >= client_sweep_ms { sweep_dead_clients(®istry); last_sweep = std::time::Instant::now(); } let idle_ms = registry.millis_since_active(); if idle_ms > max_idle_ms { - info!(idle_ms, max_idle_ms, "daemon exiting on max-idle backstop"); - break None; + sweep_dead_clients(®istry); + if backstop_should_exit(®istry, is_process_alive) { + info!(idle_ms, max_idle_ms, "daemon exiting on max-idle backstop"); + break None; + } } if registry.active_count() == 0 && idle_ms > idle_timeout_ms { info!(idle_ms, idle_timeout_ms, "daemon idle-exit: no clients"); @@ -470,6 +535,19 @@ fn sweep_dead_clients(registry: &SessionRegistry) { } } +/// #1200 max-idle-backstop decision, extracted for unit testing with an injected +/// liveness probe (mirrors `dead_session_ids`). The backstop exists only to reap +/// a PHANTOM client — one counted but gone whose socket-close was never +/// delivered. After the inactivity window, the daemon may exit only when NO +/// remaining client can be proven alive: every session is either gone (swept +/// already) or an unverifiable no-pid connection, the sole phantom class the +/// sweep cannot catch. One provably-alive client keeps the daemon up regardless +/// of how long it has been quiet, so a genuinely-live editor session that simply +/// has not queried in 30 min is never dropped to in-process mode. +fn backstop_should_exit(registry: &SessionRegistry, is_alive: impl Fn(u32) -> bool) -> bool { + !registry.has_provably_alive_client(is_alive) +} + fn start_project_watcher( project_root: &Path, options: &DaemonOptions, @@ -628,6 +706,63 @@ mod tests { assert_eq!(options.watchdog_interval, DEFAULT_WATCHDOG_INTERVAL); } + #[test] + fn backstop_exits_only_when_no_client_is_provably_alive() { + let registry = SessionRegistry::default(); + // Empty registry: no provably-alive client, so the backstop may exit. + assert!(backstop_should_exit(®istry, is_process_alive)); + + // A no-pid (unverifiable phantom) session does NOT keep the daemon up. + let phantom = registry.start_session(); + assert!(backstop_should_exit(®istry, |_| true)); + + // A provably-alive client keeps the daemon up even at the max-idle window. + let live = registry.start_session(); + registry.set_pid(live.session_id(), 1234); + assert!(!backstop_should_exit(®istry, |pid| pid == 1234)); + + // Once that client's pid is dead, the backstop may exit again. + assert!(backstop_should_exit(®istry, |pid| pid != 1234)); + + drop((phantom, live)); + } + + #[test] + fn resolve_host_pid_reads_env_and_defaults_to_none() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // SAFETY: guarded by ENV_LOCK. + unsafe { std::env::remove_var(CODEGRAPH_HOST_PPID) }; + assert_eq!(resolve_host_pid_from_env(), None); + unsafe { std::env::set_var(CODEGRAPH_HOST_PPID, "4321") }; + assert_eq!(resolve_host_pid_from_env(), Some(4321)); + unsafe { std::env::set_var(CODEGRAPH_HOST_PPID, "not-a-pid") }; + assert_eq!(resolve_host_pid_from_env(), None); + unsafe { std::env::set_var(CODEGRAPH_HOST_PPID, "0") }; + assert_eq!(resolve_host_pid_from_env(), None); + unsafe { std::env::remove_var(CODEGRAPH_HOST_PPID) }; + } + + #[test] + fn resolve_startup_handshake_timeout_parses_and_disables() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // SAFETY: guarded by ENV_LOCK. + unsafe { std::env::remove_var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS) }; + assert_eq!( + resolve_startup_handshake_timeout_ms(), + DEFAULT_STARTUP_HANDSHAKE_TIMEOUT_MS + ); + unsafe { std::env::set_var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS, "0") }; + assert_eq!(resolve_startup_handshake_timeout_ms(), 0); + unsafe { std::env::set_var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS, "1500") }; + assert_eq!(resolve_startup_handshake_timeout_ms(), 1500); + unsafe { std::env::set_var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS, "bogus") }; + assert_eq!( + resolve_startup_handshake_timeout_ms(), + DEFAULT_STARTUP_HANDSHAKE_TIMEOUT_MS + ); + unsafe { std::env::remove_var(CODEGRAPH_STARTUP_HANDSHAKE_TIMEOUT_MS) }; + } + #[cfg(unix)] #[test] fn socket_candidate_chain_puts_preferred_first_and_dedups() { diff --git a/crates/codegraph-daemon/src/session.rs b/crates/codegraph-daemon/src/session.rs index cd67ea15..b36b4b07 100644 --- a/crates/codegraph-daemon/src/session.rs +++ b/crates/codegraph-daemon/src/session.rs @@ -169,6 +169,20 @@ impl SessionRegistry { .collect() } + /// Whether ANY connected session's host pid is KNOWN and provably alive per + /// `is_alive`. A session with no announced pid is the sole class the sweep + /// cannot verify (a phantom client whose socket-close was never delivered), + /// so it never counts as provably alive. Backs the #1200 inactivity-backstop + /// gate: the daemon must keep running while at least one client is provably + /// alive, no matter how long that client has been quiet. + pub(crate) fn has_provably_alive_client(&self, is_alive: impl Fn(u32) -> bool) -> bool { + self.sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .any(|entry| matches!(entry.pid, Some(pid) if is_alive(pid))) + } + /// Force a dead session's reader to EOF by half-closing the read direction /// of its socket. The session thread, blocked in the rmcp serve loop, then sees /// EOF, returns, and drops its [`SessionGuard`] (which removes the entry + @@ -219,7 +233,7 @@ pub(crate) struct SessionGuard { } impl SessionGuard { - fn session_id(&self) -> u64 { + pub(crate) fn session_id(&self) -> u64 { self.id } } @@ -525,6 +539,30 @@ mod tests { assert_eq!(registry.active_count(), 0); } + #[test] + fn has_provably_alive_client_only_counts_known_live_pids() { + let registry = SessionRegistry::default(); + // No sessions: nothing is provably alive. + assert!(!registry.has_provably_alive_client(|_| true)); + + // A session with NO announced pid is never provably alive (the phantom + // class the sweep cannot verify) — even under an all-alive probe. + let no_pid = registry.start_session(); + assert!(!registry.has_provably_alive_client(|_| true)); + + // A session whose pid the probe reports dead is not provably alive. + let dead = registry.start_session(); + registry.set_pid(dead.session_id(), 4242); + assert!(!registry.has_provably_alive_client(|pid| pid != 4242)); + + // A session whose pid the probe reports alive IS provably alive. + let alive = registry.start_session(); + registry.set_pid(alive.session_id(), 1234); + assert!(registry.has_provably_alive_client(|pid| pid == 1234)); + + drop((no_pid, dead, alive)); + } + #[test] fn session_registry_millis_since_active_advances_over_time() { let registry = SessionRegistry::default(); diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index 7e745a35..4875f0d0 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -1130,6 +1130,18 @@ impl CodeGraphEngine { lines.push(format!("- {lang}: {count}")); } } + // #1187: only appended when the interrupted-resolution marker is set, so a + // healthy status is byte-identical to a pre-#1187 build. Tells the agent + // the blast radius is incomplete until the next sync heals it. + if self.store.is_resolution_incomplete()? { + lines.push(String::new()); + lines.push( + "**⚠ Index is PARTIAL:** a resolution pass was interrupted, so some \ + call edges are missing and the blast radius is incomplete. Run \ + `codegraph sync` to heal it." + .to_string(), + ); + } Ok(ToolResult::text(lines.join("\n"))) } diff --git a/crates/codegraph-resolve/src/resolver.rs b/crates/codegraph-resolve/src/resolver.rs index 1eaf2b33..60836297 100644 --- a/crates/codegraph-resolve/src/resolver.rs +++ b/crates/codegraph-resolve/src/resolver.rs @@ -1282,6 +1282,12 @@ impl ReferenceResolver { batch_size: usize, mut on_progress: impl FnMut(u64, u64), ) -> anyhow::Result { + // Incomplete-resolution marker (#1187): armed BEFORE the first batch and + // cleared only after the deferred passes complete below. An interrupted + // run (crash / Ctrl-C / #1122 watchdog kill) leaves it set, so the next + // `sync` knows to sweep the refs this run never reached into edges. + store.set_resolution_incomplete()?; + { let context = crate::context::StoreResolutionContext::new(store, &self.project_root); self.warm_caches(&context); @@ -1392,6 +1398,9 @@ impl ReferenceResolver { self.resolve_chained_calls_via_conformance(store)?; self.resolve_deferred_this_member_refs(store)?; + // Full pass completed: clear the #1187 marker set at the top. + store.clear_resolution_incomplete()?; + Ok(aggregate) } @@ -2361,6 +2370,39 @@ mod tests { .len() } + #[test] + fn batched_pass_clears_incomplete_marker_on_success() { + let mut store = Store::open(&temp_db("marker-clear")).expect("open"); + let add = mk_node("function:add", NodeKind::Function, "add", "math.ts"); + let caller = mk_node("function:run", NodeKind::Function, "run", "app.ts"); + store.upsert_file(&file_rec("math.ts")).expect("file"); + store.upsert_file(&file_rec("app.ts")).expect("file"); + store.upsert_nodes(&[add, caller.clone()]).expect("nodes"); + store + .insert_unresolved_refs(&[UnresolvedRef { + id: None, + from_node_id: caller.id, + reference_name: "add".to_string(), + reference_kind: EdgeKind::Calls, + line: 1, + col: 0, + candidates: None, + file_path: "app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }]) + .expect("refs"); + let mut resolver = ReferenceResolver::new("/root"); + resolver + .resolve_and_persist_batched(&mut store, 10) + .expect("batched"); + assert!( + !store.is_resolution_incomplete().expect("marker read"), + "a completed batched pass must clear the #1187 incomplete marker" + ); + } + #[test] fn resolve_and_persist_batched_reports_progress() { let mut store = Store::open(&temp_db("progress")).expect("open"); diff --git a/crates/codegraph-store/src/lib.rs b/crates/codegraph-store/src/lib.rs index b5121b34..a667970a 100644 --- a/crates/codegraph-store/src/lib.rs +++ b/crates/codegraph-store/src/lib.rs @@ -4,3 +4,6 @@ pub mod queries; pub mod schema; pub use connection::Store; +pub use queries::{ + CODEGRAPH_NO_WAL_DEFER, CODEGRAPH_WAL_VALVE_MB, DEFAULT_WAL_VALVE_MB, wal_valve_threshold_bytes, +}; diff --git a/crates/codegraph-store/src/queries.rs b/crates/codegraph-store/src/queries.rs index 9313a422..111b5fde 100644 --- a/crates/codegraph-store/src/queries.rs +++ b/crates/codegraph-store/src/queries.rs @@ -12,6 +12,36 @@ use crate::connection::Store; const SQLITE_PARAM_CHUNK_SIZE: usize = 500; +/// Env var name (#1231): set to `1` to opt out of bulk-index WAL-checkpoint +/// deferral and keep SQLite's default `wal_autocheckpoint` interval. +pub const CODEGRAPH_NO_WAL_DEFER: &str = "CODEGRAPH_NO_WAL_DEFER"; + +/// Env var name (#1231): soft WAL-growth threshold in MB that triggers the +/// bulk-index [`Store::checkpoint_wal_if_over`] fold. Non-numeric / non-positive +/// falls back to [`DEFAULT_WAL_VALVE_MB`]. +pub const CODEGRAPH_WAL_VALVE_MB: &str = "CODEGRAPH_WAL_VALVE_MB"; + +/// Default WAL-valve fold threshold in MB (#1231, mirrors upstream 256). +pub const DEFAULT_WAL_VALVE_MB: u64 = 256; + +fn wal_defer_disabled() -> bool { + matches!(std::env::var(CODEGRAPH_NO_WAL_DEFER), Ok(v) if v == "1") +} + +/// Resolve the WAL-valve fold threshold in BYTES from `CODEGRAPH_WAL_VALVE_MB` +/// (#1231); non-numeric / non-positive → [`DEFAULT_WAL_VALVE_MB`]. +pub fn wal_valve_threshold_bytes() -> u64 { + let mb = match std::env::var(CODEGRAPH_WAL_VALVE_MB) { + Ok(raw) if !raw.is_empty() => raw + .parse::() + .ok() + .filter(|&n| n > 0) + .unwrap_or(DEFAULT_WAL_VALVE_MB), + _ => DEFAULT_WAL_VALVE_MB, + }; + mb * 1024 * 1024 +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct StoreCounts { pub node_count: i64, @@ -877,24 +907,70 @@ impl Store { } /// Tune THIS connection for a from-scratch bulk index: drop `synchronous` to - /// `OFF` and grow the page cache and mmap window. This trades crash durability + /// `OFF`, grow the page cache and mmap window, and (unless opted out) DEFER WAL + /// auto-checkpointing for the whole run (#1231). This trades crash durability /// for throughput during the one-shot full index, which is always safe because /// the full-index path starts from an empty DB (`index --force` removes the file /// first) and is re-runnable from scratch on failure. /// - /// `synchronous` is a durability knob only: it never alters committed content, - /// so golden byte-equivalence is preserved. MUST be scoped to the CLI full-index - /// connection — never the shared `Store::open` defaults used by sync/daemon/watch. - /// Pair with [`Store::restore_default_pragmas`] (via a Drop guard) so the restore - /// runs on both the happy and the error path. + /// WAL deferral (`wal_autocheckpoint=0`): SQLite's default 1000-page + /// auto-checkpoint re-writes hot B-tree/FTS pages into the main DB file over and + /// over during a bulk index — the dominant disk cost on HDD-class storage. With + /// autocheckpoint off, the store becomes near-sequential WAL appends; WAL growth + /// is bounded instead by [`Store::checkpoint_wal_if_over`], and the final + /// [`Store::restore_default_pragmas`] / [`Store::compact`] folds the WAL back with + /// one `wal_checkpoint(TRUNCATE)`. Set `CODEGRAPH_NO_WAL_DEFER=1` to keep the + /// default autocheckpoint interval. + /// + /// `synchronous`, `cache_size`, `mmap_size`, and `wal_autocheckpoint` are all + /// durability/scheduling knobs only: none alters committed content, so golden + /// byte-equivalence is preserved. `wal_autocheckpoint` is a per-connection + /// setting, so a later reopen of this DB resets to the compiled default (1000). + /// MUST be scoped to the CLI full-index connection — never the shared + /// `Store::open` defaults used by sync/daemon/watch. Pair with + /// [`Store::restore_default_pragmas`] (via a Drop guard) so the restore runs on + /// both the happy and the error path. pub fn set_bulk_index_pragmas(&self) -> rusqlite::Result<()> { self.conn.pragma_update(None, "synchronous", "OFF")?; self.conn.pragma_update(None, "cache_size", -262_144)?; self.conn .pragma_update(None, "mmap_size", 1_073_741_824_i64)?; + if !wal_defer_disabled() { + self.conn.pragma_update(None, "wal_autocheckpoint", 0)?; + } Ok(()) } + /// Size in bytes of the `-wal` sidecar, or 0 when it does not exist (non-WAL + /// mode, in-memory DB, or nothing written since the last checkpoint+reset). + /// Drives the [`Store::checkpoint_wal_if_over`] valve (#1231). + pub fn wal_size_bytes(&self) -> u64 { + let mut wal = self.path().to_path_buf().into_os_string(); + wal.push("-wal"); + std::fs::metadata(wal).map(|m| m.len()).unwrap_or(0) + } + + /// WAL-valve fold (#1231): when the `-wal` sidecar has grown past + /// `threshold_bytes`, fold it back into the main DB with one + /// `wal_checkpoint(TRUNCATE)` and report `true`; otherwise a cheap no-op + /// returning `false`. Called between bulk-write batches while + /// `wal_autocheckpoint=0` is in effect, so unbounded WAL growth (which fills + /// the disk and makes every later read page through a multi-GB WAL) is capped + /// without paying the default per-1000-page checkpoint churn. TRUNCATE is + /// content-preserving — it only relocates already-committed frames — so index + /// output stays byte-identical. Safe here because the bulk index holds the + /// sole connection (no concurrent reader pins the WAL). + /// + /// Must be called with no active transaction. + pub fn checkpoint_wal_if_over(&self, threshold_bytes: u64) -> rusqlite::Result { + if self.wal_size_bytes() <= threshold_bytes { + return Ok(false); + } + self.conn + .pragma_update(None, "wal_checkpoint", "TRUNCATE")?; + Ok(true) + } + /// Undo [`Store::set_bulk_index_pragmas`]: checkpoint the WAL back into the main /// file (truncating the `-wal` sidecar) and restore `synchronous=NORMAL` so any /// later reopen of this DB sees the shared default durability. The checkpoint @@ -928,6 +1004,49 @@ impl Store { ) } + /// Delete a `project_metadata` row. Missing key is a no-op (returns 0). + /// Complements [`Self::set_project_metadata`] for flags whose ABSENCE is the + /// healthy state (see [`Self::clear_resolution_incomplete`]). + pub fn delete_project_metadata(&self, key: &str) -> rusqlite::Result { + self.conn + .execute("DELETE FROM project_metadata WHERE key = ?", [key]) + } + + /// `project_metadata` key for the incomplete-resolution MARKER (#1187). + /// + /// A full-index resolution pass SETS this at its start and DELETES it only on + /// successful completion. A crash / Ctrl-C / #1122 watchdog kill mid-resolve + /// therefore leaves the marker present, which is the signal that some + /// `unresolved_refs` were never processed into edges (the interrupted-index + /// "too-small blast radius" of the #1187 field report). This is deliberately + /// distinct from "rows at rest": Rust RETAINS unresolvable rows as live retry + /// state (#1240), so a bare row count is NOT a sound orphan signal here — the + /// marker is. + pub const RESOLUTION_INCOMPLETE_KEY: &'static str = "resolution_incomplete"; + + /// Set the incomplete-resolution marker (#1187). Called at the START of a + /// full-index resolution pass; cleared by [`Self::clear_resolution_incomplete`] + /// only when that pass completes. + pub fn set_resolution_incomplete(&self) -> rusqlite::Result { + self.set_project_metadata(Self::RESOLUTION_INCOMPLETE_KEY, "1") + } + + /// Clear the incomplete-resolution marker (#1187). Absence of the row is the + /// healthy state, so this DELETEs rather than writing a sentinel — keeping the + /// healthy `project_metadata` set identical to a pre-#1187 index. + pub fn clear_resolution_incomplete(&self) -> rusqlite::Result { + self.delete_project_metadata(Self::RESOLUTION_INCOMPLETE_KEY) + } + + /// Whether the incomplete-resolution marker is set (#1187). `true` means a + /// resolution pass was interrupted mid-run and the index is partial until the + /// next `sync` sweeps it. + pub fn is_resolution_incomplete(&self) -> rusqlite::Result { + Ok(self + .get_project_metadata(Self::RESOLUTION_INCOMPLETE_KEY)? + .is_some()) + } + /// `getStats` node-kind aggregation /// (`upstream db/queries.ts:1758-1763`): /// `SELECT kind, COUNT(*) FROM nodes GROUP BY kind`. Returned in SQLite's diff --git a/crates/codegraph-store/tests/bulk_index_pragmas.rs b/crates/codegraph-store/tests/bulk_index_pragmas.rs index 873dcab9..5a564d25 100644 --- a/crates/codegraph-store/tests/bulk_index_pragmas.rs +++ b/crates/codegraph-store/tests/bulk_index_pragmas.rs @@ -77,6 +77,96 @@ fn read_synchronous(store: &Store) -> i64 { .unwrap() } +fn read_wal_autocheckpoint(store: &Store) -> i64 { + store + .connection() + .query_row("PRAGMA wal_autocheckpoint", [], |row| row.get::<_, i64>(0)) + .unwrap() +} + +// #1231: with WAL deferral ON (default), bulk index sets wal_autocheckpoint=0 so +// SQLite stops re-writing hot pages into the main DB on a 1000-page cadence. +#[test] +fn set_bulk_index_pragmas_disables_wal_autocheckpoint_by_default() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // SAFETY: guarded by ENV_LOCK so the opt-out test's env set/remove can't race. + unsafe { std::env::remove_var("CODEGRAPH_NO_WAL_DEFER") }; + let dir = TestDir::new(); + let db_path = dir.path().join("codegraph.db"); + + let store = Store::open(&db_path).unwrap(); + assert_eq!(read_wal_autocheckpoint(&store), 1000, "default before bulk"); + + store.set_bulk_index_pragmas().unwrap(); + assert_eq!( + read_wal_autocheckpoint(&store), + 0, + "bulk index must defer WAL autocheckpoint" + ); +} + +// #1231 opt-out: CODEGRAPH_NO_WAL_DEFER=1 keeps SQLite's default autocheckpoint. +// The env-mutation window is serialized on a process-global lock so parallel +// tests cannot observe a half-set env. +#[test] +fn no_wal_defer_env_keeps_default_autocheckpoint() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let dir = TestDir::new(); + let db_path = dir.path().join("codegraph.db"); + // SAFETY: guarded by ENV_LOCK. + unsafe { std::env::set_var("CODEGRAPH_NO_WAL_DEFER", "1") }; + let store = Store::open(&db_path).unwrap(); + store.set_bulk_index_pragmas().unwrap(); + let checkpoint = read_wal_autocheckpoint(&store); + // SAFETY: guarded by ENV_LOCK. + unsafe { std::env::remove_var("CODEGRAPH_NO_WAL_DEFER") }; + assert_eq!( + checkpoint, 1000, + "opt-out must leave the default autocheckpoint interval" + ); +} + +// #1231 valve: checkpoint_wal_if_over is a no-op under the threshold and folds +// the WAL (returns true) once it grows past it. With autocheckpoint deferred, a +// batch of writes grows the -wal file, so a tiny threshold trips the fold. +#[test] +fn checkpoint_wal_if_over_folds_only_past_threshold() { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // SAFETY: guarded by ENV_LOCK so the opt-out test's env set/remove can't race. + unsafe { std::env::remove_var("CODEGRAPH_NO_WAL_DEFER") }; + let dir = TestDir::new(); + let db_path = dir.path().join("codegraph.db"); + let store = Store::open(&db_path).unwrap(); + store.set_bulk_index_pragmas().unwrap(); + + store + .connection() + .execute_batch("CREATE TABLE t(x TEXT);") + .unwrap(); + let big = "x".repeat(4096); + for _ in 0..64 { + store + .connection() + .execute("INSERT INTO t(x) VALUES (?)", [&big]) + .unwrap(); + } + + // A threshold far above the WAL size is a no-op. + assert!( + !store.checkpoint_wal_if_over(u64::MAX).unwrap(), + "must not fold when under threshold" + ); + // A zero threshold folds the (non-empty) WAL back and truncates it. + let folded = store.checkpoint_wal_if_over(0).unwrap(); + assert!(folded, "must fold when WAL exceeds threshold"); + assert!( + store.wal_size_bytes() < 4096, + "TRUNCATE must shrink the -wal sidecar after a fold" + ); +} + +static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + struct TestDir { path: PathBuf, } diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 1d163ef2..633de105 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -164,6 +164,20 @@ fn sync_paths_with_store( // Cross-file framework finalization on every sync (upstream index.ts:464). resolver.run_post_extract(store)?; } + + // Orphan sweep (#1187, upstream index.ts:645-682): a full-index resolution + // pass killed mid-run leaves the refs it never reached parked in + // unresolved_refs and sets the incomplete-resolution marker. The scoped + // incremental pass above only revisits changed files' refs, so those orphaned + // call edges stay missing until a full re-index. When the marker is present — + // INCLUDING on a no-change sync — grind the whole table down with the batched + // resolver, which heals the wedged index and clears the marker on success. A + // healthy index (marker absent) skips this entirely, so an ordinary sync is + // byte-for-byte unchanged. + if store.is_resolution_incomplete()? { + sweep_orphaned_refs(project_root, store)?; + } + outcome.duration_ms = started.elapsed().as_millis(); let mut changed_paths: Vec = reindexed.into_iter().collect(); changed_paths.sort(); @@ -221,6 +235,34 @@ fn refresh_dependent_refs( Ok(()) } +/// Grind the whole `unresolved_refs` table with the batched resolver to heal an +/// index whose full-resolution pass was interrupted (#1187 marker set). The +/// batched pass re-arms and clears the marker itself, so on success the index is +/// no longer flagged partial. Framework per-file extract is re-run first so any +/// framework refs an interrupted run never re-injected are present for the sweep. +fn sweep_orphaned_refs(project_root: &Path, store: &mut Store) -> Result<()> { + let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); + { + let context = + codegraph_resolve::StoreResolutionContext::new(store, project_root.to_string_lossy()); + resolver.initialize(&context); + } + // Framework extraction must not run in the sweep. The marker is set only + // inside the batched resolver, which runs AFTER a full index has already + // persisted its framework `unresolved_refs`, so those rows are present in the + // DB by the time the marker exists. Re-extracting them would re-inject + // duplicate rows (`unresolved_refs` has no UNIQUE constraint), which + // accumulate across interrupt→heal cycles and diverge from `index --force`. + // Re-resolving the existing rows alone heals the interrupted pass byte-equal. + resolver.resolve_and_persist_batched(store, ORPHAN_SWEEP_BATCH_ROWS)?; + resolver.run_post_extract(store)?; + Ok(()) +} + +/// Batch size for the #1187 orphan sweep, matching the CLI full-index batch +/// (`RESOLVE_BATCH_ROWS`) so a healed index is byte-equal to `index --force`. +const ORPHAN_SWEEP_BATCH_ROWS: usize = 5_000; + pub(crate) fn default_db_path(project_root: &Path) -> PathBuf { project_root.join(".codegraph").join("codegraph.db") } @@ -613,6 +655,56 @@ pub(crate) mod tests { let _ = codegraph_core::config::init_config(None, project_root); } + #[test] + fn framework_sweep_does_not_grow_unresolved_refs_across_heal_cycles() { + // Given: an indexed Godot project (autoload + signal-handler refs produce + // framework `unresolved_refs`), with the #1187 marker set so every sync + // triggers the orphan sweep. + let dir = TestDir::new("watch-godot-sweep"); + ensure_config(dir.path()); + fs::create_dir_all(dir.path().join(".codegraph")).unwrap(); + fs::write( + dir.path().join("project.godot"), + "config_version=5\n\n[application]\n\nconfig/name=\"Sweep Fixture\"\n\n[autoload]\n\nGameFlow=\"*res://game_flow.gd\"\n", + ) + .unwrap(); + fs::write( + dir.path().join("game_flow.gd"), + "extends Node\n\nfunc return_to_map() -> void:\n\tpass\n", + ) + .unwrap(); + fs::write( + dir.path().join("stage_manager.gd"), + "extends Node\n\nfunc _ready() -> void:\n\tvar button := Button.new()\n\tbutton.pressed.connect(_on_pressed.bind(button))\n\nfunc _goto_map() -> void:\n\tGameFlow.return_to_map()\n\nfunc _on_pressed(source) -> void:\n\tpass\n", + ) + .unwrap(); + let db = default_db_path(dir.path()); + sync_project_once(dir.path()).unwrap(); + + let baseline = { + let store = Store::open(&db).unwrap(); + store.set_resolution_incomplete().unwrap(); + store.unresolved_refs_count().unwrap() + }; + + // When: two bare no-change syncs run the sweep back-to-back. + sync_project_once(dir.path()).unwrap(); + let after_first = Store::open(&db).unwrap().unresolved_refs_count().unwrap(); + sync_project_once(dir.path()).unwrap(); + let after_second = Store::open(&db).unwrap().unresolved_refs_count().unwrap(); + + // Then: the row count never grows — the sweep re-resolves the existing + // framework refs instead of re-injecting duplicate ones. + assert_eq!( + after_first, baseline, + "first sweep must not grow unresolved_refs (no duplicate framework rows)" + ); + assert_eq!( + after_second, baseline, + "repeated sweeps must stay idempotent (no accumulation across heal cycles)" + ); + } + #[test] fn sync_project_once_scans_and_removes_absent_tracked_files() { // Given: a project with two source files indexed via a full scan. @@ -705,6 +797,118 @@ pub(crate) mod tests { ); } + #[test] + fn interrupted_index_heals_on_bare_sync_and_clears_marker() { + use codegraph_core::types::{EdgeKind, Language, UnresolvedRef}; + + // Given: an index whose full-resolution pass was interrupted — a + // genuinely-resolvable ref is parked in unresolved_refs with no edge, and + // the #1187 incomplete-resolution marker is set (the crash signature). + let dir = TestDir::new("watch-heal-orphan"); + fs::create_dir_all(dir.path().join("src")).unwrap(); + fs::write( + dir.path().join("src/math.ts"), + "export function add(a: number, b: number): number { return a + b; }\n", + ) + .unwrap(); + fs::write( + dir.path().join("src/app.ts"), + "import { add } from './math';\nexport function run(): number { return add(1, 2); }\n", + ) + .unwrap(); + let db = default_db_path(dir.path()); + + // Index both files so nodes exist, then simulate the interruption by + // deleting the resolved Calls edge, re-parking its unresolved ref, and + // setting the marker. + sync_changed_paths(dir.path(), &db, ["src/math.ts", "src/app.ts"]).unwrap(); + let caller_id = { + let store = Store::open(&db).unwrap(); + store + .nodes_by_file_path("src/app.ts") + .unwrap() + .into_iter() + .find(|n| n.name == "run") + .expect("run node") + .id + }; + { + let mut store = Store::open(&db).unwrap(); + store.delete_resolved_edges_from_file("src/app.ts").unwrap(); + store + .insert_unresolved_refs(&[UnresolvedRef { + id: None, + from_node_id: caller_id.clone(), + reference_name: "add".to_string(), + reference_kind: EdgeKind::Calls, + line: 2, + col: 0, + candidates: None, + file_path: "src/app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }]) + .unwrap(); + store.set_resolution_incomplete().unwrap(); + } + + // When: a bare no-change sync runs (no path is dirty on disk). + let outcome = sync_changed_paths(dir.path(), &db, Vec::::new()).unwrap(); + assert_eq!(outcome.files_reindexed, 0, "no file changed"); + + // Then: the orphaned ref is swept into a Calls edge and the marker clears. + let store = Store::open(&db).unwrap(); + assert!( + !store.is_resolution_incomplete().unwrap(), + "sweep must clear the #1187 marker" + ); + let edges = store + .edges_by_source_kind(&caller_id, Some(EdgeKind::Calls)) + .unwrap(); + assert!( + !edges.is_empty(), + "the orphaned add() call edge must be recovered by the sweep" + ); + } + + #[test] + fn healthy_sync_leaves_marker_absent_and_unresolvable_rows_parked() { + // Given: a healthy index (no marker) with a legitimately-unresolvable ref + // to an external symbol — Rust retains such rows as #1240 retry state. + let dir = TestDir::new("watch-healthy-noheal"); + fs::create_dir_all(dir.path().join("src")).unwrap(); + fs::write( + dir.path().join("src/app.ts"), + "export function run(): void { externalMissing(); }\n", + ) + .unwrap(); + let db = default_db_path(dir.path()); + sync_changed_paths(dir.path(), &db, ["src/app.ts"]).unwrap(); + + // When: a bare no-change sync runs. + let before = { + let store = Store::open(&db).unwrap(); + ( + store.is_resolution_incomplete().unwrap(), + store.unresolved_refs_count().unwrap(), + ) + }; + let outcome = sync_changed_paths(dir.path(), &db, Vec::::new()).unwrap(); + + // Then: no marker was set, nothing was swept, and the retained + // unresolvable rows are untouched (#1240 retry state preserved). + assert_eq!(outcome.files_reindexed, 0); + assert!(!before.0, "a healthy index must not carry the marker"); + let store = Store::open(&db).unwrap(); + assert!(!store.is_resolution_incomplete().unwrap()); + assert_eq!( + store.unresolved_refs_count().unwrap(), + before.1, + "unresolvable retry rows must survive a healthy sync" + ); + } + #[test] fn deleting_an_imported_module_refreshes_its_dependents() { // Given: a helper module and a consumer that resolves an import to it,