diff --git a/changelog.d/+verify-findings.bugfix.md b/changelog.d/+verify-findings.bugfix.md new file mode 100644 index 00000000..490d6004 --- /dev/null +++ b/changelog.d/+verify-findings.bugfix.md @@ -0,0 +1 @@ +A failed verification command is a `sanity` finding, or `timeout`, unless its output names a transport, executor, runtime, or resource cause, so an Lmod `error:` line no longer makes it `compile`. Finding summaries and campaign history name the class and exit status as the state file does, `sanity` and `exit 1`, not `Sanity` and `exit Some(1)`. diff --git a/docs/orgmode/reference/campaigns.org b/docs/orgmode/reference/campaigns.org index 8c514c2d..f3617c98 100644 --- a/docs/orgmode/reference/campaigns.org +++ b/docs/orgmode/reference/campaigns.org @@ -143,6 +143,11 @@ classification so the finding reflects the actual compiler or runtime error. | ~timeout~ | retryable | Routed timeout or scheduler wall time | | ~unknown~ | requires-judgment | Unclassified evidence; inspect the exact command and nested log | +A failed verification command is ~sanity~, or ~timeout~ when it ran out of +time, unless its output names a transport, executor, runtime, or resource +cause. Nothing is compiled while it runs, so an Lmod ~error:~ line is a module +that does not load, not a ~compile~ failure. + The class narrows the first repair surface; it does not replace diagnosis. For example, a SAT-compatible dependency that fails to compile remains a ~compile~ finding until evidence supports a stack exclusion or package patch. diff --git a/llms-full.txt b/llms-full.txt index 5591be69..da568179 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -10116,6 +10116,11 @@ classification so the finding reflects the actual compiler or runtime error. | `timeout` | retryable | Routed timeout or scheduler wall time | | `unknown` | requires-judgment | Unclassified evidence; inspect the exact command and nested log | +A failed verification command is `sanity`, or `timeout` when it ran out of +time, unless its output names a transport, executor, runtime, or resource +cause. Nothing is compiled while it runs, so an Lmod `error:` line is a module +that does not load, not a `compile` failure. + The class narrows the first repair surface; it does not replace diagnosis. For example, a SAT-compatible dependency that fails to compile remains a `compile` finding until evidence supports a stack exclusion or package patch. diff --git a/src/campaign.rs b/src/campaign.rs index 8fd0e025..f693e7b2 100644 --- a/src/campaign.rs +++ b/src/campaign.rs @@ -90,6 +90,15 @@ pub enum BuildFindingClass { Unknown, } +/// The name the state file uses, so a summary or history line says +/// `dependency-missing` where the JSON does. +impl std::fmt::Display for BuildFindingClass { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = serde_json::to_value(self).map_err(|_| std::fmt::Error)?; + formatter.write_str(value.as_str().unwrap_or("unknown")) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] /// What kind of intervention a finding needs. @@ -423,7 +432,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result Result Result Result Result FindingDisposition { } fn finding_summary(class: BuildFindingClass, exit_code: Option) -> String { - let class = serde_json::to_value(class) - .ok() - .and_then(|value| value.as_str().map(str::to_string)) - .unwrap_or_else(|| "unknown".into()); + format!( + "{class} failure from EasyBuild command ({})", + exit_status(exit_code) + ) +} + +fn exit_status(exit_code: Option) -> String { match exit_code { - Some(code) => format!("{class} failure from EasyBuild command (exit {code})"), - None => format!("{class} failure from EasyBuild command (no exit status)"), + Some(code) => format!("exit {code}"), + None => "no exit status".into(), } } diff --git a/tests/campaign.rs b/tests/campaign.rs index c1be30fb..6311bac3 100644 --- a/tests/campaign.rs +++ b/tests/campaign.rs @@ -986,6 +986,169 @@ fn failed_profile_verification_preserves_the_build_claim_and_finding() { .contains("module=QMCPACK/4.3.0-complex-foss-2026.1")); } +/// Lmod 8.7 after an EESSI 2026.06 init, when the module is not on +/// `MODULEPATH`. +const LMOD_UNKNOWN_MODULE: &str = "\ +Module for EESSI/2026.06 loaded successfully +Lmod has detected the following error: The following module(s) are unknown: +\"natsort/8.4.0-GCCcore-15.2.0\" + +Please check the spelling or version number. Also try \"module spider ...\" +It is also possible your cache file is out-of-date; it may help to try: + $ module --ignore_cache load \"natsort/8.4.0-GCCcore-15.2.0\" +"; + +#[test] +fn a_failed_verification_command_is_a_sanity_finding() { + let sanity = [ + LMOD_UNKNOWN_MODULE, + "Traceback (most recent call last):\n File \"\", line 1, in \nModuleNotFoundError: No module named 'natsort'\n", + "hello.c:3:5: error: implicit declaration of function 'MPI_Init'\ncompilation terminated.\n", + "ctest: 3 tests failed\n", + "pip install failed\n", + ]; + for stderr in sanity { + assert_eq!( + classify_build_failure("verify", "", stderr, Some(1)), + BuildFindingClass::Sanity, + "{stderr}" + ); + } + let target_side = [ + ( + "ssh: connect to host failed", + Some(255), + BuildFindingClass::Transport, + ), + ( + "qmcpack: command not found\nNo such file or directory", + Some(127), + BuildFindingClass::Runtime, + ), + ( + "slurmstepd: error: Detected 1 oom-kill", + Some(137), + BuildFindingClass::Resource, + ), + ("", Some(124), BuildFindingClass::Timeout), + ]; + for (stderr, exit_code, expected) in target_side { + assert_eq!( + classify_build_failure("verify", "", stderr, exit_code), + expected, + "{stderr}" + ); + } +} + +#[test] +fn a_failed_verification_finding_reads_the_way_the_state_file_spells_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let bundle = temp.path().join("bundle"); + let recipes = bundle.join("easyconfigs/n/natsort"); + std::fs::create_dir_all(&recipes).expect("recipes"); + std::fs::create_dir_all(bundle.join("locks")).expect("locks"); + let verify = format!( + "printf '%s' '{}' >&2; exit 1", + LMOD_UNKNOWN_MODULE.replace('\'', "") + ); + std::fs::write( + bundle.join("package.plan.json"), + serde_json::json!({ + "package": {"name": "natsort", "version": "8.4.0"}, + "build": {"toolchain": {"name": "GCCcore", "version": "15.2.0"}}, + "profiles": [{ + "name": "default", + "default": true, + "versionsuffix": [], + "verification_commands": [{"program": "sh", "args": ["-c", verify]}] + }] + }) + .to_string(), + ) + .expect("manifest"); + std::fs::write( + bundle.join("locks/default.lock.json"), + r#"{"profile":"default","solver":"resolvo"}"#, + ) + .expect("lock"); + write_valid_recipe(&recipes.join("natsort.eb"), "natsort", "8.4.0"); + + let state = run_campaign(&CampaignRequest { + bundle, + target: target("true"), + state_path: temp.path().join("campaign.json"), + }) + .expect("campaign"); + assert_eq!(state.status, CampaignStatus::Failed); + let finding = &state.findings[0]; + assert_eq!(finding.class, BuildFindingClass::Sanity); + assert_eq!(finding.disposition, FindingDisposition::RequiresJudgment); + assert_eq!( + finding.summary, + "binary verification failed for profile default (exit 1)" + ); + let details: Vec<&str> = state + .history + .iter() + .map(|event| event.detail.as_str()) + .collect(); + assert!( + details + .iter() + .any(|detail| detail.contains("classified binary verification failure as sanity")), + "{details:#?}" + ); +} + +#[test] +fn a_failed_build_names_its_class_the_way_the_state_file_does() { + let temp = tempfile::tempdir().expect("tempdir"); + let bundle = temp.path().join("bundle"); + let recipes = bundle.join("easyconfigs/e/eOn"); + std::fs::create_dir_all(&recipes).expect("recipes"); + std::fs::create_dir_all(bundle.join("locks")).expect("locks"); + std::fs::write( + bundle.join("package.plan.json"), + r#"{"package":{"name":"eOn","version":"2.16.0"}}"#, + ) + .expect("manifest"); + std::fs::write( + bundle.join("locks/default.lock.json"), + r#"{"profile":"default","solver":"resolvo"}"#, + ) + .expect("lock"); + write_valid_recipe(&recipes.join("eOn.eb"), "eOn", "2.16.0"); + let command = temp.path().join("fake-eb"); + write_executable( + &command, + "#!/bin/sh\nprintf '%s\\n' 'ERROR: Missing dependencies: Boost/1.92.0-gfbf-2026.1 (no easyconfig file or existing module found)'\nexit 1\n", + ) + .expect("command"); + + let state = run_campaign(&CampaignRequest { + bundle, + target: target(command.to_str().expect("command path")), + state_path: temp.path().join("campaign.json"), + }) + .expect("campaign"); + assert_eq!( + state.findings[0].class, + BuildFindingClass::DependencyMissing + ); + let details: Vec<&str> = state + .history + .iter() + .map(|event| event.detail.as_str()) + .collect(); + assert!( + details + .iter() + .any(|detail| detail.contains("classified build failure as dependency-missing")), + "{details:#?}" + ); +} + #[test] fn campaign_cli_claims_and_resolves_findings_for_omp_workers() { let temp = tempfile::tempdir().expect("tempdir");