Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/+verify-findings.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A failed verification command is a `sanity` finding, or `timeout`, unless its output names a transport, executor, runtime, or resource cause, so an Lmod `error:` line no longer makes it `compile`. Finding summaries and campaign history name the class and exit status as the state file does, `sanity` and `exit 1`, not `Sanity` and `exit Some(1)`.
5 changes: 5 additions & 0 deletions docs/orgmode/reference/campaigns.org
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@ classification so the finding reflects the actual compiler or runtime error.
| ~timeout~ | retryable | Routed timeout or scheduler wall time |
| ~unknown~ | requires-judgment | Unclassified evidence; inspect the exact command and nested log |

A failed verification command is ~sanity~, or ~timeout~ when it ran out of
time, unless its output names a transport, executor, runtime, or resource
cause. Nothing is compiled while it runs, so an Lmod ~error:~ line is a module
that does not load, not a ~compile~ failure.

The class narrows the first repair surface; it does not replace diagnosis.
For example, a SAT-compatible dependency that fails to compile remains a
~compile~ finding until evidence supports a stack exclusion or package patch.
Expand Down
5 changes: 5 additions & 0 deletions llms-full.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10116,6 +10116,11 @@ classification so the finding reflects the actual compiler or runtime error.
| `timeout` | retryable | Routed timeout or scheduler wall time |
| `unknown` | requires-judgment | Unclassified evidence; inspect the exact command and nested log |

A failed verification command is `sanity`, or `timeout` when it ran out of
time, unless its output names a transport, executor, runtime, or resource
cause. Nothing is compiled while it runs, so an Lmod `error:` line is a module
that does not load, not a `compile` failure.

The class narrows the first repair surface; it does not replace diagnosis.
For example, a SAT-compatible dependency that fails to compile remains a
`compile` finding until evidence supports a stack exclusion or package patch.
Expand Down
52 changes: 36 additions & 16 deletions src/campaign.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,15 @@ pub enum BuildFindingClass {
Unknown,
}

/// The name the state file uses, so a summary or history line says
/// `dependency-missing` where the JSON does.
impl std::fmt::Display for BuildFindingClass {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let value = serde_json::to_value(self).map_err(|_| std::fmt::Error)?;
formatter.write_str(value.as_str().unwrap_or("unknown"))
}
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
/// What kind of intervention a finding needs.
Expand Down Expand Up @@ -423,7 +432,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result<CampaignState, Campaign
recipe: Some(recipe_text),
detail: stuck_detail(
&state,
format!("classified packaging preflight failure as {class:?}"),
format!("classified packaging preflight failure as {class}"),
),
});
write_state(&request.state_path, &state)?;
Expand Down Expand Up @@ -469,7 +478,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result<CampaignState, Campaign
recipe: None,
detail: stuck_detail(
&state,
"classified bundle staging failure as Transport".to_string(),
"classified bundle staging failure as transport".to_string(),
),
});
write_state(&request.state_path, &state)?;
Expand Down Expand Up @@ -623,7 +632,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result<CampaignState, Campaign
attempt: state.attempts,
status: CampaignStatus::Failed,
recipe: Some(recipe_text),
detail: stuck_detail(&state, format!("classified build failure as {class:?}")),
detail: stuck_detail(&state, format!("classified build failure as {class}")),
});
write_state(&request.state_path, &state)?;
return Ok(state);
Expand Down Expand Up @@ -700,9 +709,9 @@ pub fn run_campaign(request: &CampaignRequest) -> Result<CampaignState, Campaign
recipe: format!("profile:{}", profile.name),
target: run_target.name.clone(),
summary: format!(
"binary verification failed for profile {} (exit {:?})",
"binary verification failed for profile {} ({})",
profile.name,
output.status.code()
exit_status(output.status.code())
),
evidence: format!("module={module}\n{}", compact_evidence(&stdout, &stderr)),
command,
Expand All @@ -721,7 +730,7 @@ pub fn run_campaign(request: &CampaignRequest) -> Result<CampaignState, Campaign
recipe: Some(format!("profile:{}", profile.name)),
detail: stuck_detail(
&state,
format!("classified binary verification failure as {class:?}"),
format!("classified binary verification failure as {class}"),
),
});
write_state(&request.state_path, &state)?;
Expand Down Expand Up @@ -1050,6 +1059,16 @@ pub fn classify_build_failure(
BuildFindingClass::DependencyMissing
} else if (text.contains("glibc_") && text.contains("not found")) || missing_executable {
BuildFindingClass::Runtime
} else if stage.eq_ignore_ascii_case("verify") {
// A profile's verification command runs the installed module, so
// nothing in its output is a fetch, patch, configure, compile or
// install step failing: Lmod's `error:` line is the module not
// loading.
if text.contains("timed out") || exit_code == Some(124) {
BuildFindingClass::Timeout
} else {
BuildFindingClass::Sanity
}
} else if checksum_failure {
BuildFindingClass::Checksum
} else if source_failure {
Expand Down Expand Up @@ -1085,8 +1104,6 @@ pub fn classify_build_failure(
BuildFindingClass::Install
} else if text.contains("timed out") || exit_code == Some(124) {
BuildFindingClass::Timeout
} else if stage.eq_ignore_ascii_case("verify") {
BuildFindingClass::Sanity
} else {
BuildFindingClass::Unknown
}
Expand Down Expand Up @@ -1143,7 +1160,7 @@ fn record_target_command_failure(
stage: stage.into(),
recipe: recipe.into(),
target: target.name.clone(),
summary: format!("{class:?} target command could not start"),
summary: format!("{class} target command could not start"),
evidence: error.to_string(),
command,
exit_code: None,
Expand All @@ -1159,7 +1176,7 @@ fn record_target_command_failure(
attempt: state.attempts,
status: CampaignStatus::Failed,
recipe: Some(recipe.into()),
detail: format!("classified target command failure as {class:?}"),
detail: format!("classified target command failure as {class}"),
});
}

Expand Down Expand Up @@ -1274,13 +1291,16 @@ fn disposition(class: BuildFindingClass) -> FindingDisposition {
}

fn finding_summary(class: BuildFindingClass, exit_code: Option<i32>) -> String {
let class = serde_json::to_value(class)
.ok()
.and_then(|value| value.as_str().map(str::to_string))
.unwrap_or_else(|| "unknown".into());
format!(
"{class} failure from EasyBuild command ({})",
exit_status(exit_code)
)
}

fn exit_status(exit_code: Option<i32>) -> String {
match exit_code {
Some(code) => format!("{class} failure from EasyBuild command (exit {code})"),
None => format!("{class} failure from EasyBuild command (no exit status)"),
Some(code) => format!("exit {code}"),
None => "no exit status".into(),
}
}

Expand Down
163 changes: 163 additions & 0 deletions tests/campaign.rs
Original file line number Diff line number Diff line change
Expand Up @@ -986,6 +986,169 @@ fn failed_profile_verification_preserves_the_build_claim_and_finding() {
.contains("module=QMCPACK/4.3.0-complex-foss-2026.1"));
}

/// Lmod 8.7 after an EESSI 2026.06 init, when the module is not on
/// `MODULEPATH`.
const LMOD_UNKNOWN_MODULE: &str = "\
Module for EESSI/2026.06 loaded successfully
Lmod has detected the following error: The following module(s) are unknown:
\"natsort/8.4.0-GCCcore-15.2.0\"

Please check the spelling or version number. Also try \"module spider ...\"
It is also possible your cache file is out-of-date; it may help to try:
$ module --ignore_cache load \"natsort/8.4.0-GCCcore-15.2.0\"
";

#[test]
fn a_failed_verification_command_is_a_sanity_finding() {
let sanity = [
LMOD_UNKNOWN_MODULE,
"Traceback (most recent call last):\n File \"<string>\", line 1, in <module>\nModuleNotFoundError: No module named 'natsort'\n",
"hello.c:3:5: error: implicit declaration of function 'MPI_Init'\ncompilation terminated.\n",
"ctest: 3 tests failed\n",
"pip install failed\n",
];
for stderr in sanity {
assert_eq!(
classify_build_failure("verify", "", stderr, Some(1)),
BuildFindingClass::Sanity,
"{stderr}"
);
}
let target_side = [
(
"ssh: connect to host failed",
Some(255),
BuildFindingClass::Transport,
),
(
"qmcpack: command not found\nNo such file or directory",
Some(127),
BuildFindingClass::Runtime,
),
(
"slurmstepd: error: Detected 1 oom-kill",
Some(137),
BuildFindingClass::Resource,
),
("", Some(124), BuildFindingClass::Timeout),
];
for (stderr, exit_code, expected) in target_side {
assert_eq!(
classify_build_failure("verify", "", stderr, exit_code),
expected,
"{stderr}"
);
}
}

#[test]
fn a_failed_verification_finding_reads_the_way_the_state_file_spells_it() {
let temp = tempfile::tempdir().expect("tempdir");
let bundle = temp.path().join("bundle");
let recipes = bundle.join("easyconfigs/n/natsort");
std::fs::create_dir_all(&recipes).expect("recipes");
std::fs::create_dir_all(bundle.join("locks")).expect("locks");
let verify = format!(
"printf '%s' '{}' >&2; exit 1",
LMOD_UNKNOWN_MODULE.replace('\'', "")
);
std::fs::write(
bundle.join("package.plan.json"),
serde_json::json!({
"package": {"name": "natsort", "version": "8.4.0"},
"build": {"toolchain": {"name": "GCCcore", "version": "15.2.0"}},
"profiles": [{
"name": "default",
"default": true,
"versionsuffix": [],
"verification_commands": [{"program": "sh", "args": ["-c", verify]}]
}]
})
.to_string(),
)
.expect("manifest");
std::fs::write(
bundle.join("locks/default.lock.json"),
r#"{"profile":"default","solver":"resolvo"}"#,
)
.expect("lock");
write_valid_recipe(&recipes.join("natsort.eb"), "natsort", "8.4.0");

let state = run_campaign(&CampaignRequest {
bundle,
target: target("true"),
state_path: temp.path().join("campaign.json"),
})
.expect("campaign");
assert_eq!(state.status, CampaignStatus::Failed);
let finding = &state.findings[0];
assert_eq!(finding.class, BuildFindingClass::Sanity);
assert_eq!(finding.disposition, FindingDisposition::RequiresJudgment);
assert_eq!(
finding.summary,
"binary verification failed for profile default (exit 1)"
);
let details: Vec<&str> = state
.history
.iter()
.map(|event| event.detail.as_str())
.collect();
assert!(
details
.iter()
.any(|detail| detail.contains("classified binary verification failure as sanity")),
"{details:#?}"
);
}

#[test]
fn a_failed_build_names_its_class_the_way_the_state_file_does() {
let temp = tempfile::tempdir().expect("tempdir");
let bundle = temp.path().join("bundle");
let recipes = bundle.join("easyconfigs/e/eOn");
std::fs::create_dir_all(&recipes).expect("recipes");
std::fs::create_dir_all(bundle.join("locks")).expect("locks");
std::fs::write(
bundle.join("package.plan.json"),
r#"{"package":{"name":"eOn","version":"2.16.0"}}"#,
)
.expect("manifest");
std::fs::write(
bundle.join("locks/default.lock.json"),
r#"{"profile":"default","solver":"resolvo"}"#,
)
.expect("lock");
write_valid_recipe(&recipes.join("eOn.eb"), "eOn", "2.16.0");
let command = temp.path().join("fake-eb");
write_executable(
&command,
"#!/bin/sh\nprintf '%s\\n' 'ERROR: Missing dependencies: Boost/1.92.0-gfbf-2026.1 (no easyconfig file or existing module found)'\nexit 1\n",
)
.expect("command");

let state = run_campaign(&CampaignRequest {
bundle,
target: target(command.to_str().expect("command path")),
state_path: temp.path().join("campaign.json"),
})
.expect("campaign");
assert_eq!(
state.findings[0].class,
BuildFindingClass::DependencyMissing
);
let details: Vec<&str> = state
.history
.iter()
.map(|event| event.detail.as_str())
.collect();
assert!(
details
.iter()
.any(|detail| detail.contains("classified build failure as dependency-missing")),
"{details:#?}"
);
}

#[test]
fn campaign_cli_claims_and_resolves_findings_for_omp_workers() {
let temp = tempfile::tempdir().expect("tempdir");
Expand Down
Loading