From e53b39a8182366e5ec42ccdf735efd6d3041fe3a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 16 Nov 2026 02:01:35 +0000 Subject: [PATCH] fix(campaign): classify verification failures as sanity A profile verification command that failed with an Lmod "error:" line was classed as compile, because the generic "error:" test came before the verify-stage fallback to sanity, which was never reached. reference/campaigns.org maps a failed verification command to sanity. Nothing is fetched, patched, configured, compiled, linked or installed while a verification command runs, so after the target-side classes (transport, executor, resource, runtime) and the robot classes a verify-stage failure is now timeout or sanity. The finding summary printed the exit code with Rust debug formatting, "exit Some(1)", and every history line named the class the same way, "as Compile" or "as DependencyMissing", where the state file says "compile" and "dependency-missing". BuildFindingClass now displays as its serde name, and the verification summary uses the exit-status text the build summary already used. Tests: the classifier on recorded Lmod and Python output for the verify stage, and campaigns whose verification command or EasyBuild command fails, checked for class, summary and history wording. --- changelog.d/+verify-findings.bugfix.md | 1 + docs/orgmode/reference/campaigns.org | 5 + llms-full.txt | 5 + src/campaign.rs | 52 +++++--- tests/campaign.rs | 163 +++++++++++++++++++++++++ 5 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 changelog.d/+verify-findings.bugfix.md diff --git a/changelog.d/+verify-findings.bugfix.md b/changelog.d/+verify-findings.bugfix.md new file mode 100644 index 00000000..490d6004 --- /dev/null +++ b/changelog.d/+verify-findings.bugfix.md @@ -0,0 +1 @@ +A failed verification command is a `sanity` finding, or `timeout`, unless its output names a transport, executor, runtime, or resource cause, so an Lmod `error:` line no longer makes it `compile`. Finding summaries and campaign history name the class and exit status as the state file does, `sanity` and `exit 1`, not `Sanity` and `exit Some(1)`. diff --git a/docs/orgmode/reference/campaigns.org b/docs/orgmode/reference/campaigns.org index 8c514c2d..f3617c98 100644 --- a/docs/orgmode/reference/campaigns.org +++ b/docs/orgmode/reference/campaigns.org @@ -143,6 +143,11 @@ classification so the finding reflects the actual compiler or runtime error. | ~timeout~ | retryable | Routed timeout or scheduler wall time | | ~unknown~ | requires-judgment | Unclassified evidence; inspect the exact command and nested log | +A failed verification command is ~sanity~, or ~timeout~ when it ran out of +time, unless its output names a transport, executor, runtime, or resource +cause. Nothing is compiled while it runs, so an Lmod ~error:~ line is a module +that does not load, not a ~compile~ failure. + The class narrows the first repair surface; it does not replace diagnosis. For example, a SAT-compatible dependency that fails to compile remains a ~compile~ finding until evidence supports a stack exclusion or package patch. diff --git a/llms-full.txt b/llms-full.txt index 5591be69..da568179 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -10116,6 +10116,11 @@ classification so the finding reflects the actual compiler or runtime error. | `timeout` | retryable | Routed timeout or scheduler wall time | | `unknown` | requires-judgment | Unclassified evidence; inspect the exact command and nested log | +A failed verification command is `sanity`, or `timeout` when it ran out of +time, unless its output names a transport, executor, runtime, or resource +cause. Nothing is compiled while it runs, so an Lmod `error:` line is a module +that does not load, not a `compile` failure. + The class narrows the first repair surface; it does not replace diagnosis. For example, a SAT-compatible dependency that fails to compile remains a `compile` finding until evidence supports a stack exclusion or package patch. diff --git a/src/campaign.rs b/src/campaign.rs index 8fd0e025..f693e7b2 100644 --- a/src/campaign.rs +++ b/src/campaign.rs @@ -90,6 +90,15 @@ pub enum BuildFindingClass { Unknown, } +/// The name the state file uses, so a summary or history line says +/// `dependency-missing` where the JSON does. +impl std::fmt::Display for BuildFindingClass { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = serde_json::to_value(self).map_err(|_| std::fmt::Error)?; + formatter.write_str(value.as_str().unwrap_or("unknown")) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] /// What kind of intervention a finding needs. @@ -423,7 +432,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result Result Result Result Result FindingDisposition { } fn finding_summary(class: BuildFindingClass, exit_code: Option) -> String { - let class = serde_json::to_value(class) - .ok() - .and_then(|value| value.as_str().map(str::to_string)) - .unwrap_or_else(|| "unknown".into()); + format!( + "{class} failure from EasyBuild command ({})", + exit_status(exit_code) + ) +} + +fn exit_status(exit_code: Option) -> String { match exit_code { - Some(code) => format!("{class} failure from EasyBuild command (exit {code})"), - None => format!("{class} failure from EasyBuild command (no exit status)"), + Some(code) => format!("exit {code}"), + None => "no exit status".into(), } } diff --git a/tests/campaign.rs b/tests/campaign.rs index c1be30fb..6311bac3 100644 --- a/tests/campaign.rs +++ b/tests/campaign.rs @@ -986,6 +986,169 @@ fn failed_profile_verification_preserves_the_build_claim_and_finding() { .contains("module=QMCPACK/4.3.0-complex-foss-2026.1")); } +/// Lmod 8.7 after an EESSI 2026.06 init, when the module is not on +/// `MODULEPATH`. +const LMOD_UNKNOWN_MODULE: &str = "\ +Module for EESSI/2026.06 loaded successfully +Lmod has detected the following error: The following module(s) are unknown: +\"natsort/8.4.0-GCCcore-15.2.0\" + +Please check the spelling or version number. Also try \"module spider ...\" +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load \"natsort/8.4.0-GCCcore-15.2.0\" +"; + +#[test] +fn a_failed_verification_command_is_a_sanity_finding() { + let sanity = [ + LMOD_UNKNOWN_MODULE, + "Traceback (most recent call last):\n File \"\", line 1, in \nModuleNotFoundError: No module named 'natsort'\n", + "hello.c:3:5: error: implicit declaration of function 'MPI_Init'\ncompilation terminated.\n", + "ctest: 3 tests failed\n", + "pip install failed\n", + ]; + for stderr in sanity { + assert_eq!( + classify_build_failure("verify", "", stderr, Some(1)), + BuildFindingClass::Sanity, + "{stderr}" + ); + } + let target_side = [ + ( + "ssh: connect to host failed", + Some(255), + BuildFindingClass::Transport, + ), + ( + "qmcpack: command not found\nNo such file or directory", + Some(127), + BuildFindingClass::Runtime, + ), + ( + "slurmstepd: error: Detected 1 oom-kill", + Some(137), + BuildFindingClass::Resource, + ), + ("", Some(124), BuildFindingClass::Timeout), + ]; + for (stderr, exit_code, expected) in target_side { + assert_eq!( + classify_build_failure("verify", "", stderr, exit_code), + expected, + "{stderr}" + ); + } +} + +#[test] +fn a_failed_verification_finding_reads_the_way_the_state_file_spells_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let bundle = temp.path().join("bundle"); + let recipes = bundle.join("easyconfigs/n/natsort"); + std::fs::create_dir_all(&recipes).expect("recipes"); + std::fs::create_dir_all(bundle.join("locks")).expect("locks"); + let verify = format!( + "printf '%s' '{}' >&2; exit 1", + LMOD_UNKNOWN_MODULE.replace('\'', "") + ); + std::fs::write( + bundle.join("package.plan.json"), + serde_json::json!({ + "package": {"name": "natsort", "version": "8.4.0"}, + "build": {"toolchain": {"name": "GCCcore", "version": "15.2.0"}}, + "profiles": [{ + "name": "default", + "default": true, + "versionsuffix": [], + "verification_commands": [{"program": "sh", "args": ["-c", verify]}] + }] + }) + .to_string(), + ) + .expect("manifest"); + std::fs::write( + bundle.join("locks/default.lock.json"), + r#"{"profile":"default","solver":"resolvo"}"#, + ) + .expect("lock"); + write_valid_recipe(&recipes.join("natsort.eb"), "natsort", "8.4.0"); + + let state = run_campaign(&CampaignRequest { + bundle, + target: target("true"), + state_path: temp.path().join("campaign.json"), + }) + .expect("campaign"); + assert_eq!(state.status, CampaignStatus::Failed); + let finding = &state.findings[0]; + assert_eq!(finding.class, BuildFindingClass::Sanity); + assert_eq!(finding.disposition, FindingDisposition::RequiresJudgment); + assert_eq!( + finding.summary, + "binary verification failed for profile default (exit 1)" + ); + let details: Vec<&str> = state + .history + .iter() + .map(|event| event.detail.as_str()) + .collect(); + assert!( + details + .iter() + .any(|detail| detail.contains("classified binary verification failure as sanity")), + "{details:#?}" + ); +} + +#[test] +fn a_failed_build_names_its_class_the_way_the_state_file_does() { + let temp = tempfile::tempdir().expect("tempdir"); + let bundle = temp.path().join("bundle"); + let recipes = bundle.join("easyconfigs/e/eOn"); + std::fs::create_dir_all(&recipes).expect("recipes"); + std::fs::create_dir_all(bundle.join("locks")).expect("locks"); + std::fs::write( + bundle.join("package.plan.json"), + r#"{"package":{"name":"eOn","version":"2.16.0"}}"#, + ) + .expect("manifest"); + std::fs::write( + bundle.join("locks/default.lock.json"), + r#"{"profile":"default","solver":"resolvo"}"#, + ) + .expect("lock"); + write_valid_recipe(&recipes.join("eOn.eb"), "eOn", "2.16.0"); + let command = temp.path().join("fake-eb"); + write_executable( + &command, + "#!/bin/sh\nprintf '%s\\n' 'ERROR: Missing dependencies: Boost/1.92.0-gfbf-2026.1 (no easyconfig file or existing module found)'\nexit 1\n", + ) + .expect("command"); + + let state = run_campaign(&CampaignRequest { + bundle, + target: target(command.to_str().expect("command path")), + state_path: temp.path().join("campaign.json"), + }) + .expect("campaign"); + assert_eq!( + state.findings[0].class, + BuildFindingClass::DependencyMissing + ); + let details: Vec<&str> = state + .history + .iter() + .map(|event| event.detail.as_str()) + .collect(); + assert!( + details + .iter() + .any(|detail| detail.contains("classified build failure as dependency-missing")), + "{details:#?}" + ); +} + #[test] fn campaign_cli_claims_and_resolves_findings_for_omp_workers() { let temp = tempfile::tempdir().expect("tempdir");