diff --git a/changelog.d/+admit-refusal.bugfix.md b/changelog.d/+admit-refusal.bugfix.md new file mode 100644 index 00000000..6d6a336f --- /dev/null +++ b/changelog.d/+admit-refusal.bugfix.md @@ -0,0 +1 @@ +A job `stack admit` refuses for want of inodes carries a `refusal`: the inodes it is short, then the `available`, `reserved`, `footprint` and `margin` that decided it. It was `null`, so only a failed quota or scheduler command said why. diff --git a/changelog.d/+ledger-show-stage.bugfix.md b/changelog.d/+ledger-show-stage.bugfix.md new file mode 100644 index 00000000..fd90565f --- /dev/null +++ b/changelog.d/+ledger-show-stage.bugfix.md @@ -0,0 +1 @@ +`ledger show` prints each record's `stage` after `action`, so the started and finished records of one staged rebuild no longer print as two identical rows. diff --git a/docs/orgmode/explanation/sharding-admission.org b/docs/orgmode/explanation/sharding-admission.org index cb598832..64f3398d 100644 --- a/docs/orgmode/explanation/sharding-admission.org +++ b/docs/orgmode/explanation/sharding-admission.org @@ -140,8 +140,9 @@ reservation whose job is absent from the live list is stale: ignored, and deleted only with ~--drop-stale~. Exit 0 writes ~.json~ (~job~, ~footprint~, ~margin~, ~ts~) under -~--reservations~. Exit 78 writes nothing. A quota command or a scheduler -command that fails also exits 78, and the JSON ~refusal~ field says why. +~--reservations~. Exit 78 writes nothing, and the JSON ~refusal~ field +says why: the inodes the job is short, or the quota or scheduler command +that failed. A reservation file that is unreadable exits 1, because a reservation that cannot be parsed would hide inodes. @@ -164,7 +165,7 @@ eb-stack stack admit \ #+end_src #+begin_example -{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} +{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"351000 inodes short: 100000 available, 0 reserved, footprint 351000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} #+end_example The process exits 78. @@ -214,7 +215,7 @@ eb-stack stack admit \ #+end_src #+begin_example -{"admitted":false,"available":0,"counted":[],"footprint":200000,"grace":"expired","hard":3500000,"headroom":-300000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":true,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":3100000} +{"admitted":false,"available":0,"counted":[],"footprint":200000,"grace":"expired","hard":3500000,"headroom":-300000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"300000 inodes short: 0 available, 0 reserved, footprint 200000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":true,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":3100000} #+end_example The process exits 78. diff --git a/docs/orgmode/get-started.org b/docs/orgmode/get-started.org index 5477560e..ce7cb96a 100644 --- a/docs/orgmode/get-started.org +++ b/docs/orgmode/get-started.org @@ -101,7 +101,7 @@ eb-stack stack admit \ #+end_src #+begin_example -{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} +{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"351000 inodes short: 100000 available, 0 reserved, footprint 351000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} #+end_example The exit status is 78. Exit statuses for the rest of the CLI are in the diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 62d7185f..8bcbb02f 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -1076,8 +1076,12 @@ is a file name: letters, digits, ~.~, ~_~, and ~-~. The command prints one JSON line (~job~, ~admitted~, ~available~, ~used~, ~soft~, ~hard~, ~reserved~, ~footprint~, ~margin~, ~headroom~, ~live_jobs~, ~counted~, ~stale~, ~stale_dropped~, ~unknown~, ~unknown_reserved~, ~refusal~, -~reservation~, ~ts~) and exits 0 when admitted, 78 when refused. The -footprint model is +~reservation~, ~ts~) and exits 0 when admitted, 78 when refused. +~refusal~ is ~null~ on admit. On a refusal it names the command that +failed, or the inodes the job is short followed by the ~available~, +~reserved~, ~footprint~ and ~margin~ that decided it: +~106000 inodes short: 1000 available, 2000 reserved, footprint 5000, +margin 100000~. The footprint model is [[file:../explanation/sharding-admission.org][Sharding and inode admission]]. A reservation file that is not JSON, has no ~job~ or ~footprint~, or names another job than its file name exits 1, because a reservation @@ -2067,8 +2071,9 @@ through a symbolic or hard link, is refused, and so is a head path that is a symbolic link. A signing key with more than one hard link is refused. The lock is a POSIX record lock: whether it excludes writers on several hosts depends on the shared file system's lock support. ~show~ prints ~index~, -~ts~, ~action~, ~target~, ~module~ and ~actor~ per record, newest first, then -~K of N records~. ~--package~ matches a whole module name, or the software +~ts~, ~action~, ~stage~, ~target~, ~module~ and ~actor~ per record, newest +first, then ~K of N records~; ~stage~ is empty for a record without one. +~--package~ matches a whole module name, or the software name before its ~/~. ~--json~ prints the records as a JSON array. ~verify --record N --head HEAD~ checks HEAD's signature against diff --git a/llms-full.txt b/llms-full.txt index 5591be69..baddf7c5 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -200,7 +200,7 @@ eb-stack stack admit \ ``` ``` -{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} +{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"351000 inodes short: 100000 available, 0 reserved, footprint 351000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} ``` The exit status is 78. Exit statuses for the rest of the CLI are in the @@ -7735,8 +7735,12 @@ is a file name: letters, digits, `.`, `_`, and `-`. The command prints one JSON line (`job`, `admitted`, `available`, `used`, `soft`, `hard`, `reserved`, `footprint`, `margin`, `headroom`, `live_jobs`, `counted`, `stale`, `stale_dropped`, `unknown`, `unknown_reserved`, `refusal`, -`reservation`, `ts`) and exits 0 when admitted, 78 when refused. The -footprint model is +`reservation`, `ts`) and exits 0 when admitted, 78 when refused. +`refusal` is `null` on admit. On a refusal it names the command that +failed, or the inodes the job is short followed by the `available`, +`reserved`, `footprint` and `margin` that decided it: +~106000 inodes short: 1000 available, 2000 reserved, footprint 5000, +margin 100000~. The footprint model is [Sharding and inode admission](https://eb-stack.rgoswami.me/../explanation/sharding-admission.html.md). A reservation file that is not JSON, has no `job` or `footprint`, or names another job than its file name exits 1, because a reservation @@ -8726,8 +8730,9 @@ through a symbolic or hard link, is refused, and so is a head path that is a symbolic link. A signing key with more than one hard link is refused. The lock is a POSIX record lock: whether it excludes writers on several hosts depends on the shared file system's lock support. `show` prints `index`, -`ts`, `action`, `target`, `module` and `actor` per record, newest first, then -`K of N records`. `--package` matches a whole module name, or the software +`ts`, `action`, `stage`, `target`, `module` and `actor` per record, newest +first, then `K of N records`; `stage` is empty for a record without one. +`--package` matches a whole module name, or the software name before its `/`. `--json` prints the records as a JSON array. `verify --record N --head HEAD` checks HEAD's signature against @@ -11238,8 +11243,9 @@ reservation whose job is absent from the live list is stale: ignored, and deleted only with `--drop-stale`. Exit 0 writes `.json` (`job`, `footprint`, `margin`, `ts`) under -`--reservations`. Exit 78 writes nothing. A quota command or a scheduler -command that fails also exits 78, and the JSON `refusal` field says why. +`--reservations`. Exit 78 writes nothing, and the JSON `refusal` field +says why: the inodes the job is short, or the quota or scheduler command +that failed. A reservation file that is unreadable exits 1, because a reservation that cannot be parsed would hide inodes. @@ -11262,7 +11268,7 @@ eb-stack stack admit \ ``` ``` -{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} +{"admitted":false,"available":100000,"counted":[],"footprint":351000,"grace":null,"hard":3500000,"headroom":-351000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"351000 inodes short: 100000 available, 0 reserved, footprint 351000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":false,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":2900000} ``` The process exits 78. @@ -11312,7 +11318,7 @@ eb-stack stack admit \ ``` ``` -{"admitted":false,"available":0,"counted":[],"footprint":200000,"grace":"expired","hard":3500000,"headroom":-300000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":null,"reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":true,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":3100000} +{"admitted":false,"available":0,"counted":[],"footprint":200000,"grace":"expired","hard":3500000,"headroom":-300000,"job":"lane-1","live_jobs":null,"live_jobs_cmd":null,"lock":null,"lock_kind":null,"margin":100000,"refusal":"300000 inodes short: 0 available, 0 reserved, footprint 200000, margin 100000","reservation":null,"reserved":0,"soft":3000000,"soft_exceeded":true,"stale":[],"stale_dropped":false,"ts":"2026-11-10T12:00:00Z","unknown":[],"unknown_reserved":0,"used":3100000} ``` The process exits 78. diff --git a/src/inode_budget.rs b/src/inode_budget.rs index 7942307f..9e880fe1 100644 --- a/src/inode_budget.rs +++ b/src/inode_budget.rs @@ -1134,6 +1134,15 @@ pub fn run_stack_admit(args: &AdmitRequest) -> Result { ); if refusal.is_some() { decision.admitted = false; + } else if !decision.admitted { + refusal = Some(format!( + "{} inodes short: {} available, {} reserved, footprint {}, margin {}", + decision.headroom.unsigned_abs(), + decision.available, + decision.reserved, + decision.footprint, + decision.margin + )); } if args.drop_stale { for stale in &decision.stale { diff --git a/src/ledger.rs b/src/ledger.rs index fcfe8a5a..fe8746f7 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -1505,15 +1505,23 @@ pub fn show(request: &ShowRequest) -> Result { text.push('\n'); text } else { - let mut text = String::from("index\tts\taction\ttarget\tmodule\tactor\n"); + let mut text = String::from("index\tts\taction\tstage\ttarget\tmodule\tactor\n"); for entry in &selected { - let field = |name: &str| entry.record[name].as_str().unwrap_or_default().to_string(); + let field = |name: &str| { + entry + .record + .get(name) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; let _ = writeln!( text, - "{}\t{}\t{}\t{}\t{}\t{}", + "{}\t{}\t{}\t{}\t{}\t{}\t{}", entry.index, field("ts"), field("action"), + field("stage"), field("target"), field("module"), field("actor") diff --git a/tests/ledger.rs b/tests/ledger.rs index a3c24bfc..369f498f 100644 --- a/tests/ledger.rs +++ b/tests/ledger.rs @@ -243,7 +243,7 @@ fn append_and_verify_round_trip() { "PETSc", ])); let rows = show.lines().collect::>(); - assert_eq!(rows[0], "index\tts\taction\ttarget\tmodule\tactor"); + assert_eq!(rows[0], "index\tts\taction\tstage\ttarget\tmodule\tactor"); assert!(rows[1].starts_with("2\t"), "newest first: {show}"); assert!(rows[2].starts_with("1\t"), "{show}"); assert_eq!(rows[3], "2 of 3 records"); @@ -261,6 +261,52 @@ fn append_and_verify_round_trip() { assert_eq!(records[0]["index"], 1); } +#[test] +fn show_tells_a_started_record_from_its_finished_one() { + let fixture = fixture(); + for (n, stage) in ["started", "finished"].iter().enumerate() { + let mut value = record("pigz/2.8-GCCcore-15.2.0", "eb-staged-rebuild/7"); + value["action"] = json!("rebuild-inplace"); + value["stage"] = json!(stage); + append( + &fixture, + &value, + &fixture.work.join(format!("head-{n}.json")), + ); + } + append( + &fixture, + &record("zlib/1.3.1-GCCcore-15.2.0", "jenkins/run/7"), + &fixture.work.join("head-2.json"), + ); + let show = ok(&eb(&["ledger", "show", "--log", s(&fixture.log)])); + let rows = show + .lines() + .map(|row| row.split('\t').collect::>()) + .collect::>(); + assert_eq!( + rows[0], + ["index", "ts", "action", "stage", "target", "module", "actor"], + "{show}" + ); + assert_eq!( + rows[1][..4], + ["2", "2026-11-03T10:00:00Z", "install", ""], + "{show}" + ); + assert_eq!( + rows[2][..4], + ["1", "2026-11-03T10:00:00Z", "rebuild-inplace", "finished"], + "{show}" + ); + assert_eq!( + rows[3][..4], + ["0", "2026-11-03T10:00:00Z", "rebuild-inplace", "started"], + "{show}" + ); + assert_eq!(rows[2][4..], rows[3][4..], "{show}"); +} + #[test] fn a_head_signed_by_another_key_fails() { let fixture = fixture(); diff --git a/tests/stack_admit.rs b/tests/stack_admit.rs index b72f9849..2a8e1637 100644 --- a/tests/stack_admit.rs +++ b/tests/stack_admit.rs @@ -101,6 +101,40 @@ fn a_lane_is_admitted_then_the_next_is_refused_and_a_stale_reservation_is_droppe assert_eq!(again["reserved"], 61_000); } +#[test] +fn a_job_the_inodes_cannot_hold_is_refused_with_the_shortfall() { + let tmp = tempfile::tempdir().unwrap(); + let quota = tmp.path().join("quota.json"); + // 900,000 soft minus 899,000 used: 1,000 available. + std::fs::write( + "a, + r#"{"inodes_used": 899000, "inodes_soft": 900000, "inodes_hard": 1000000}"#, + ) + .unwrap(); + let reservations = tmp.path().join("reservations"); + std::fs::create_dir_all(&reservations).unwrap(); + std::fs::write( + reservations.join("4420.json"), + r#"{"job": "4420", "footprint": 2000, "ts": "2026-11-06T00:00:00Z"}"#, + ) + .unwrap(); + + // 1,000 - 2,000 - 5,000 - 100,000 = -106,000. + let refused = admit("a, &reservations, "4421", "5000", Some("4420")); + assert_eq!(refused["admitted"], false, "{refused}"); + assert_eq!(refused["headroom"], -106_000); + assert_eq!( + refused["refusal"], + "106000 inodes short: 1000 available, 2000 reserved, footprint 5000, margin 100000", + "{refused}" + ); + + std::fs::write("a, r#"{"inodes_available": 500000}"#).unwrap(); + let admitted = admit("a, &reservations, "4421", "5000", Some("4420")); + assert_eq!(admitted["admitted"], true, "{admitted}"); + assert_eq!(admitted["refusal"], Value::Null, "{admitted}"); +} + #[test] fn the_quota_may_come_from_stdin_and_a_bad_job_id_is_a_usage_error() { use std::io::Write;