diff --git a/cli/cli/src/main.rs b/cli/cli/src/main.rs index 76bd478fc6..a57c089a76 100644 --- a/cli/cli/src/main.rs +++ b/cli/cli/src/main.rs @@ -327,7 +327,7 @@ fn run() -> i32 { } } }; - dx_process::configure_real(&workspace); + dx_process::configure_real_in(&workspace, dx_plan::slots::slot_idle_secs(&workspace)); dx_plan::plan::set_rules_dx_repo(&workspace); if let Some(note) = dx_plan::args::bare_scope_note(&invocation, &workspace, &cwd) { eprintln!("{note}"); diff --git a/cli/plan/src/slots.rs b/cli/plan/src/slots.rs index 7c3cd83f74..59c23412cf 100644 --- a/cli/plan/src/slots.rs +++ b/cli/plan/src/slots.rs @@ -30,6 +30,45 @@ pub const BUILD_SLOTS_DISABLED_CODE: &str = "build_slots_disabled"; /// Notice code for the disk reclaim a build slot runs before a build. pub const DISK_LOW_CODE: &str = "disk_low"; +/// Seconds a `dx worktree` slot's Bazel server may sit idle before it exits. +pub const SLOT_MAX_IDLE_SECS: u64 = 300; + +/// The `--max_idle_secs` for the server of `workspace`: short inside a `dx worktree` slot, `None` elsewhere. +pub fn slot_idle_secs(workspace: &Path) -> Option { + dx_worktree::lease_for_dir(workspace).map(|_| SLOT_MAX_IDLE_SECS) +} + +/// Sets the memory one build slot is planned to use: MiB, or a count with an `m` or `g` suffix. +pub const SLOT_MEMORY_ENV: &str = "DX_BUILD_SLOT_MEMORY"; + +/// Reads a slot memory size in bytes; `None` for zero or anything unreadable. +pub fn parse_slot_memory(value: &str) -> Option { + let value = value.trim().to_ascii_lowercase(); + let (digits, unit) = match value.chars().last()? { + 'g' => (&value[..value.len() - 1], 30), + 'm' => (&value[..value.len() - 1], 20), + _ => (value.as_str(), 20), + }; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let bytes = digits.parse::().ok()?.checked_mul(1 << unit)?; + (bytes > 0).then_some(bytes) +} + +/// Memory one local build slot is planned to use: `DX_BUILD_SLOT_MEMORY` when set and valid. +pub fn slot_bytes(env_get: &dyn Fn(&str) -> Option) -> u64 { + env_get(SLOT_MEMORY_ENV) + .and_then(|value| parse_slot_memory(&value)) + .unwrap_or(BYTES_PER_SLOT) +} + +/// `bytes` rescaled so that dividing by the default slot size counts slots of `per_slot` bytes. +fn per_slot_scaled(bytes: u64, per_slot: u64) -> u64 { + let scaled = u128::from(bytes) * u128::from(BYTES_PER_SLOT) / u128::from(per_slot); + u64::try_from(scaled).unwrap_or(u64::MAX) +} + pub mod pressure; use pressure::{HostPressure, NoSignal, PRESSURE_WAIT, Pressure, PressurePause}; @@ -345,7 +384,12 @@ pub fn share_report( let dir = slots_dir(env_get)?; let memory = available_memory()?; let reserved = nested_reserve(workspace, env_get); - let size = sizing(setting, memory, reserved)?; + let per_slot = slot_bytes(env_get); + let size = sizing( + setting, + per_slot_scaled(memory, per_slot), + per_slot_scaled(reserved, per_slot), + )?; let cpus = std::thread::available_parallelism().map_or(1, |cpus| cpus.get()); let cpus = u64::try_from(cpus).unwrap_or(1); // `usize::MAX` is no slot, so `held` counts every held slot plus the new run. @@ -1125,6 +1169,7 @@ pub struct Gate { reclaimer: Option, reaper: Option, wait: Option, + slot_bytes: u64, } /// The time source a gate paces its pressure pause by. @@ -1186,6 +1231,15 @@ impl Gate { reclaimer: None, reaper: None, wait: None, + slot_bytes: BYTES_PER_SLOT, + } + } + + /// Plans each local slot to use `bytes` of memory. + pub fn slot_memory(self, bytes: u64) -> Self { + Gate { + slot_bytes: bytes, + ..self } } @@ -1214,7 +1268,7 @@ impl Gate { let target = if self.remote { BYTES_PER_REMOTE_SLOT } else { - BYTES_PER_SLOT + self.slot_bytes }; Some((reaper.0)(&keep, target)) } @@ -1309,11 +1363,16 @@ impl Gate { let dir = slots_dir(env_get)?; let memory = available_memory()?; let reserved = nested_reserve(workspace, env_get); + let per_slot = slot_bytes(env_get); let Sizing { count, pool, extras, - } = sizing(setting, memory, reserved)?; + } = sizing( + setting, + per_slot_scaled(memory, per_slot), + per_slot_scaled(reserved, per_slot), + )?; let read = dx_process::rc::read_file; let read_user_rc = dx_process::RcPolicy::from_env(env_get, workspace, &read).read_user_rc; let startup = own_startup_options(args); @@ -1348,11 +1407,12 @@ impl Gate { warm, ..Gate::new(dir, count, workspace.display().to_string(), json) .reserving(pool) + .slot_memory(per_slot) .extra(extras, remote) .with_pressure(Arc::new(HostPressure::new(if remote { BYTES_PER_REMOTE_SLOT } else { - BYTES_PER_SLOT + per_slot }))) .sharing(cpus, memory, reserved) .reclaiming(move || dx_worktree::reclaim_for_build(&reclaim_in)) @@ -1412,7 +1472,9 @@ impl Gate { remote: self.remote, pause: PressurePause::new(PRESSURE_WAIT), now: &*self.clock.0, - deadline: self.wait.and_then(|wait| (self.clock.0)().checked_add(wait)), + deadline: self + .wait + .and_then(|wait| (self.clock.0)().checked_add(wait)), }; let result = acquire_reserving( &self.dir, @@ -2627,6 +2689,28 @@ mod tests { } } + #[test] + fn slot_memory_parses_sizes_and_scales_counts() { + assert_eq!(parse_slot_memory("4g"), Some(4 << 30)); + assert_eq!(parse_slot_memory(" 512M "), Some(512 << 20)); + assert_eq!(parse_slot_memory("2048"), Some(2048 << 20)); + for bad in ["", "0", "g", "-1g", "1.5g", "4x"] { + assert_eq!(parse_slot_memory(bad), None, "{bad}"); + } + let env = |value: &'static str| { + move |name: &str| (name == SLOT_MEMORY_ENV).then(|| value.to_owned()) + }; + assert_eq!(slot_bytes(&env("6g")), 6 << 30); + assert_eq!(slot_bytes(&env("nope")), BYTES_PER_SLOT); + assert_eq!(slot_bytes(&|_: &str| None), BYTES_PER_SLOT); + let memory = per_slot_scaled(24 << 30, 6 << 30); + assert_eq!(slot_count(memory, 0), 4); + assert_eq!( + slot_count(per_slot_scaled(24 << 30, 1 << 30), 0), + MAX_SLOTS.min(24) + ); + } + #[test] fn available_memory_text_parses() { let text = "MemTotal: 4096 kB\nMemAvailable: 2048 kB\n"; diff --git a/cli/process/src/lib.rs b/cli/process/src/lib.rs index 926bcdc52e..c8e1846434 100644 --- a/cli/process/src/lib.rs +++ b/cli/process/src/lib.rs @@ -33,8 +33,8 @@ mod stderr_tail; pub use rc::{ DISK_CACHE_GC_MAX_AGE, DISK_CACHE_GC_MAX_SIZE, LocalShare, PRESET_LOCAL_RESOURCES, RcColor, RcPolicy, ShareToken, StartupPaths, VerbMask, build_verb_options, clear_share, - clear_share_source, configure, configure_real, current, default_disk_cache, repository_caches, - set_share, set_share_source, share, startup_output_paths, user_cache_dir, + clear_share_source, configure, configure_real, configure_real_in, current, default_disk_cache, + repository_caches, set_share, set_share_source, share, startup_output_paths, user_cache_dir, }; pub use runner::{BinaryRunner, Holding}; pub use stderr_tail::last_bazel_stderr; @@ -397,11 +397,14 @@ pub fn head_settings(repo: &str) -> Vec { fn launcher_head(policy: &RcPolicy, startup_options: &[String]) -> Vec { let opts = policy.startup_opts(); - let mut argv = Vec::with_capacity(1 + opts.len() + startup_options.len()); + let mut argv = Vec::with_capacity(2 + opts.len() + startup_options.len()); argv.push(launcher_argv0().to_owned()); for opt in opts { argv.push((*opt).to_owned()); } + if let Some(secs) = policy.slot_idle_secs { + argv.push(format!("--max_idle_secs={secs}")); + } argv.extend(startup_options.iter().cloned()); argv } diff --git a/cli/process/src/rc.rs b/cli/process/src/rc.rs index 885138407a..5abba59798 100644 --- a/cli/process/src/rc.rs +++ b/cli/process/src/rc.rs @@ -175,6 +175,8 @@ pub struct RcPolicy { pub disk_cache_gc: bool, /// The gc idle delay dx adds so cleanup can run before an idle server exits. pub gc_idle_delay_secs: Option, + /// The `--max_idle_secs` dx starts a worktree slot's server with. + pub slot_idle_secs: Option, /// Where an rc file sets `--color`, which then wins over `NO_COLOR`. pub rc_color: RcColor, } @@ -188,6 +190,7 @@ static HERMETIC: RcPolicy = RcPolicy { local_memory: VerbMask::ALL, disk_cache_gc: false, gc_idle_delay_secs: None, + slot_idle_secs: None, rc_color: RcColor::NONE, }; @@ -288,6 +291,16 @@ impl RcPolicy { env_get: &dyn Fn(&str) -> Option, workspace: &Path, read_file: &dyn Fn(&Path) -> Option, + ) -> RcPolicy { + Self::from_env_in(env_get, workspace, read_file, None) + } + + /// `from_env` for a workspace whose server idles out after `slot_idle_secs`, when given. + pub fn from_env_in( + env_get: &dyn Fn(&str) -> Option, + workspace: &Path, + read_file: &dyn Fn(&Path) -> Option, + slot_idle_secs: Option, ) -> RcPolicy { let in_bazel_test = env_get("BAZEL_TEST").is_some_and(|value| !value.is_empty()); if in_bazel_test { @@ -318,11 +331,17 @@ impl RcPolicy { }; let disk_cache_gc = disk_cache.is_some() && !rc_sets_disk_cache_gc(&roots, workspace, read_file); + let slot_idle_secs = slot_idle_secs.filter(|secs| *secs > 0 && !ci); let gc_idle_delay_secs = if ci || rc_sets_flag(&roots, workspace, read_file, GC_IDLE_DELAY_FLAG) { None } else { - rc_min_max_idle_secs(&roots, workspace, read_file).and_then(gc_idle_delay_for) + let rc_idle = rc_min_max_idle_secs(&roots, workspace, read_file); + let idle = match (rc_idle, slot_idle_secs) { + (Some(rc), Some(slot)) => Some(rc.min(slot)), + (rc, slot) => rc.or(slot), + }; + idle.and_then(gc_idle_delay_for) }; RcPolicy { read_user_rc: true, @@ -332,6 +351,7 @@ impl RcPolicy { local_memory: VerbMask::where_rc_leaves_free(&lines, "memory"), disk_cache_gc, gc_idle_delay_secs, + slot_idle_secs, rc_color: RcColor::from_lines(&lines), } } @@ -438,10 +458,16 @@ pub fn configure(policy: RcPolicy) { /// Sets the policy for this process from the real environment and files. pub fn configure_real(workspace: &Path) { - configure(RcPolicy::from_env( + configure_real_in(workspace, None); +} + +/// `configure_real` for a workspace whose server idles out after `slot_idle_secs`, when given. +pub fn configure_real_in(workspace: &Path, slot_idle_secs: Option) { + configure(RcPolicy::from_env_in( &|name| std::env::var(name).ok(), workspace, &read_file, + slot_idle_secs, )); } diff --git a/cli/process/src/rc_tests.rs b/cli/process/src/rc_tests.rs index b1c6eac773..1059e12317 100644 --- a/cli/process/src/rc_tests.rs +++ b/cli/process/src/rc_tests.rs @@ -338,6 +338,7 @@ fn local_policy() -> RcPolicy { local_memory: VerbMask::ALL, disk_cache_gc: true, gc_idle_delay_secs: None, + slot_idle_secs: None, rc_color: RcColor::NONE, } } @@ -1186,3 +1187,24 @@ fn an_rc_color_choice_counts_only_when_its_section_and_config_are_active() { assert!(chosen(nested, &["bazel", "test", "--config=outer"])); assert!(!chosen("test --color=yes\n", &["bazel", "build", "//..."])); } + +#[test] +fn a_slot_idle_adds_a_startup_flag_and_an_idle_delay() { + let policy = RcPolicy::from_env_in( + &|name| (name == "HOME").then(|| "/home/u".to_owned()), + Path::new("/ws"), + &|_| None, + Some(300), + ); + assert_eq!(policy.slot_idle_secs, Some(300)); + assert_eq!(policy.gc_idle_delay_secs, Some(100)); + let head = startup_argv_with(&policy, &[]); + assert_eq!(head.last().map(String::as_str), Some("--max_idle_secs=300")); + let plain = RcPolicy::from_env_in(&|_| None, Path::new("/ws"), &|_| None, None); + assert_eq!(plain.slot_idle_secs, None); + assert!( + !startup_argv_with(&plain, &[]) + .iter() + .any(|a| a.starts_with("--max_idle_secs")) + ); +} diff --git a/cli/worktree/src/hook.rs b/cli/worktree/src/hook.rs index 8b77949929..ace82df52f 100644 --- a/cli/worktree/src/hook.rs +++ b/cli/worktree/src/hook.rs @@ -315,7 +315,7 @@ pub fn ignored(kind: HookKind, input: &Value) -> bool { && Dialect::of(input) != Dialect::Claude } -/// Marks the slots of a finished subagent as held by a gone holder. +/// Marks the slots of a finished subagent as held by a gone holder and stops their Bazel servers. pub fn stop( pool: &Pool, host: &dyn Host, @@ -327,7 +327,7 @@ pub fn stop( } let agent = field(input, "agent_id").ok_or_else(|| missing(HookKind::Stop, "agent_id"))?; let session = who.session.clone().or_else(|| field(input, "session_id")); - let _lock = pool.lock()?; + let lock = pool.lock()?; let mut marked = Vec::new(); for mut record in pool.records() { if record.held @@ -341,6 +341,10 @@ pub fn stop( marked.push(record.slot); } } + drop(lock); + for slot in &marked { + ops::stop_server(host, &pool.slot_path(*slot)); + } Ok(marked) } diff --git a/cli/worktree/src/tests.rs b/cli/worktree/src/tests.rs index 4aba40dd9d..ba563a8082 100644 --- a/cli/worktree/src/tests.rs +++ b/cli/worktree/src/tests.rs @@ -3628,10 +3628,12 @@ mod hooks { .is_empty() ); let stop = serde_json::json!({"agent_id": "a1"}); + host.serve(900, &pool.slot_path(1)); assert_eq!( hook::stop(&pool, &host, &first, &stop).expect("stop"), vec![1] ); + assert_eq!(host.shutdown_paths(), vec![pool.slot_path(1)]); let row = ops::list(&pool, &host, false).expect("list").rows.remove(0); assert_eq!( row.state, diff --git a/docs/cli/commands/README.md b/docs/cli/commands/README.md index 3d97608f81..32da666566 100644 --- a/docs/cli/commands/README.md +++ b/docs/cli/commands/README.md @@ -100,7 +100,8 @@ When `CLAUDECODE`, `CODEX_THREAD_ID`, `CURSOR_AGENT`, `GEMINI_CLI`, `AI_AGENT`, Commands that run Bazel wait for a build slot, so parallel runs on one machine do not run out of memory. Slots are shared by every worktree on the machine. The count is the machine memory, or the cgroup limit if lower, divided by 3 GiB. -It is at least 1 and at most 16. +It is at least 1 and at most 16. `DX_BUILD_SLOT_MEMORY` replaces the 3 GiB: +MiB, or a number ending in `m` or `g`. An unreadable or zero value is ignored. ```sh tools/dx build //... --build-slots 4 @@ -150,6 +151,7 @@ A flag on the command line wins over these defaults. | `DX_OUTPUT=` | Default for `--output`. | | `DX_BUILD_SLOTS=auto\|\|off` | Default for `--build-slots`. | | `DX_BUILD_SLOTS_WAIT=` | Default for `--build-slots-wait`. | +| `DX_BUILD_SLOT_MEMORY=` | Memory planned per build slot, default `3g`. | | `NO_COLOR=` | Disable color in `auto` mode. | | `BUILD_WORKSPACE_DIRECTORY=` | Workspace start under `bazel run`. | | `BUILD_WORKING_DIRECTORY=` | Current directory under `bazel run`, for `--here` and a relative `worktree release `. | diff --git a/docs/cli/commands/worktree.md b/docs/cli/commands/worktree.md index 18925b5b9d..f351c04cdc 100644 --- a/docs/cli/commands/worktree.md +++ b/docs/cli/commands/worktree.md @@ -63,6 +63,11 @@ Another holder's slot is freed only when it is reclaimable or with `--force`. It `slot_busy` while a `dx` command runs in the slot. On an external worktree of this repository it removes the checkout and keeps the branch. +Commands that run Bazel inside a slot start its server with +`--max_idle_secs=300`, so an idle slot's server exits after five minutes. +`--build-slots ` limits how many run at once and `DX_BUILD_SLOT_MEMORY` sets +the memory each is planned to use. + The working-directory check reads `/proc` on Linux only. It does not see other users' processes, processes hidden by `hidepid` or processes in other PID namespaces, and each check of a held slot scans every process. Any process whose @@ -168,7 +173,7 @@ Cursor or Gemini CLI. They ignore `DX_*` variables and print text. | --- | --- | --- | | `create` | `name`, `cwd` | Leases a slot on branch `worktree-` and prints the path. | | `remove` | `worktree_path` | Releases that slot. Paths outside the pool are ignored. | -| `stop` | `agent_id` | Marks the slots that agent holds as held by a gone holder. | +| `stop` | `agent_id` | Marks the slots that agent holds as held by a gone holder and stops their Bazel servers. | | `guard` | `tool_input.command` | Exits `2` for `git worktree add`, `remove` or `prune`. Exits `2` for `git checkout`, `switch`, `reset`, `rebase`, `fetch`, `pull`, `stash` or `commit`, `bazel`, and `dx build`, `test` or `check` in a slot another agent holds. Exits `2` for the same commands in the primary checkout when the caller holds a slot; the message names the slot. Read-only git, `gh` and `dx worktree add`, `release` and `list` stay allowed. | `create`, `remove` and `stop` exit `1` on any failure, including empty or