diff --git a/docs/compatibility.md b/docs/compatibility.md index f844b70..5f2d8a5 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -89,10 +89,10 @@ feature inventory ([`inventory.md`](inventory.md)), and — where reproducible the pinned upstream compiler. - **Fixture headers.** Every `.del`/`.ostw` corpus fixture carries - `// source: `, `// license: MIT`, and `// expect: ` + `// source: `, `// license: `, and `// expect: ` directives. The corpus harness (`tests/corpus.rs`, run on every CI run) - fails on missing source/license directives and asserts each fixture's - declared outcome. + fails on missing or empty source/license directives and asserts each + fixture's declared outcome. - **Accept/reject agreement.** The primary compatibility record is accept/reject and diagnostic-presence agreement per fixture, expressed as `// expect:` outcomes — never output-text identity. @@ -110,6 +110,12 @@ the pinned upstream compiler. inconclusive evidence. Unknown expectations require an explicit non-passing status; they are never promoted to compatibility by native agreement. +`pinned-oracle` fixtures must use the pinned OSTW compiler repository and +commit. `real-project` fixtures must use their own immutable repository, +revision, path, and license provenance; they cannot reuse the pinned upstream +compiler identity. Compiler-shipped Examples and Modules remain project-level +`pinned-oracle` fixtures until an independent real project is added. + ## Pinned upstream oracle and provenance boundary Compatibility is defined against a single pinned upstream reference, diff --git a/docs/workshop-conformance.md b/docs/workshop-conformance.md index a7cb534..9ca0cd2 100644 --- a/docs/workshop-conformance.md +++ b/docs/workshop-conformance.md @@ -16,10 +16,23 @@ Each fixture has an independent evidence classification: | `semantic-contract` | The expectation is defined by a documented DEL/OSTW semantic contract. | | `internal-invariant` | The assertion is explicitly about a del-rs representation invariant, not upstream compatibility. | +For `pinned-oracle` cases, the report requires the `// source:` URL to point at +the repository and commit recorded in the support matrix. For `real-project` +cases, the report requires a separate repository, a full 40-hex-digit commit, +and a non-empty source path. A real-project case cannot use the pinned +upstream compiler repository as its provenance. Every fixture also requires a +non-empty `// license:` marker. + +The current complete project fixtures come from the pinned OSTW compiler +repository and are therefore explicitly classified as `pinned-oracle`. They +provide project-level import/semantic/HIR coverage, but are not independent +real-project evidence. An independent real-project fixture remains a visible +follow-up for #26. + Existing fixtures with a pinned `// source:` URL are classified as -`pinned-oracle` automatically; files under `tests/corpus/projects/` are -classified as `real-project`. A fixture may override this with -`// evidence: ...`. +`pinned-oracle` automatically; files under `tests/corpus/projects/` default to +`real-project` but must opt into `pinned-oracle` when they are compiler-owned +fixtures. A fixture may override this with `// evidence: ...`. The report separates `matched`, `known-gaps`, `unsupported`, `unexpected-regressions`, and `inconclusive`. An `unknown` fixture must declare @@ -30,6 +43,10 @@ Optional `// matrix: feature.id, ...` directives link a source case to the DEL/OSTW support matrix. They are validated against `docs/support-matrix.toml` without copying Workshop catalog definitions. +Project fixtures are checked at two complementary levels: the report runs each +source entry through project loading, semantic analysis, and HIR validation, +while the project test also checks the complete import graph as one project. + ## Workshop integration When `workshop-rs#10` publishes canonical feature identities, an integration diff --git a/src/compatibility.rs b/src/compatibility.rs index 6172551..e2faeb7 100644 --- a/src/compatibility.rs +++ b/src/compatibility.rs @@ -5,11 +5,11 @@ //! or it is reported as a known gap/inconclusive case; an implementation that //! happens to agree with an unproven case cannot turn that case into a pass. +use crate::SourceMap; use crate::diagnostics::Phase; use crate::matrix; -use crate::project::{load_project, ProjectOptions}; +use crate::project::{ProjectOptions, load_project}; use crate::syntax::parse_source; -use crate::SourceMap; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -89,7 +89,8 @@ pub const REPORT_SCHEMA: u32 = 1; /// Load and execute every source fixture in `tests/corpus`. pub fn run(root: &Path) -> Result> { - let fixtures = discover(root)?; + let matrix = matrix::load_and_validate().map_err(|problems| problems)?; + let fixtures = discover(root, &matrix)?; let mut cases = Vec::with_capacity(fixtures.len()); for fixture in fixtures { cases.push(evaluate(root, fixture)); @@ -111,23 +112,22 @@ pub fn run(root: &Path) -> Result> { Ok(CompatibilityReport { schema: REPORT_SCHEMA, - upstream_pin: matrix::load() - .map_err(|e| vec![format!("support matrix does not parse: {e}")])? - .meta - .upstream_pin, + upstream_pin: matrix.meta.upstream_pin.clone(), summary, cases, }) } -fn discover(root: &Path) -> Result, Vec> { +fn discover( + root: &Path, + matrix: &matrix::SupportMatrix, +) -> Result, Vec> { let mut paths = Vec::new(); visit(&root.join("tests/corpus"), &mut paths).map_err(|e| vec![e.to_string()])?; paths.sort(); let mut errors = Vec::new(); let mut fixtures = Vec::with_capacity(paths.len()); - let matrix = matrix::load_and_validate().map_err(|problems| problems)?; for path in paths { let text = match fs::read_to_string(&path) { Ok(text) => text, @@ -136,7 +136,7 @@ fn discover(root: &Path) -> Result, Vec> { continue; } }; - match parse_metadata(root, &path, &text, &matrix) { + match parse_metadata(root, &path, &text, matrix) { Ok(fixture) => fixtures.push(fixture), Err(error) => errors.push(format!("{}: {error}", path.display())), } @@ -207,6 +207,7 @@ fn parse_metadata( let source = source.ok_or("missing // source: independent evidence pointer")?; let license = license.ok_or("missing // license: provenance marker")?; + validate_license(&license)?; let expect = expect.ok_or("missing // expect: outcome")?; let evidence = match evidence { Some(evidence) => evidence, @@ -216,6 +217,8 @@ fn parse_metadata( })?, }; + validate_source(&source, evidence, &matrix.meta)?; + for id in &matrix_ids { if !matrix.entries.iter().any(|feature| feature.id == *id) { return Err(format!("unknown // matrix: feature id {id}")); @@ -228,13 +231,13 @@ fn parse_metadata( FixtureStatus::KnownGap | FixtureStatus::Unsupported | FixtureStatus::Inconclusive, ) => {} Some(other) => { - return Err(format!("unknown outcome cannot be classified as {other:?}")) + return Err(format!("unknown outcome cannot be classified as {other:?}")); } None => { return Err( "unknown outcome requires // status: known-gap | unsupported | inconclusive" .into(), - ) + ); } } } else if status.is_some() { @@ -257,6 +260,84 @@ fn parse_metadata( }) } +fn validate_source( + source: &str, + evidence: EvidenceSource, + meta: &matrix::MatrixMeta, +) -> Result<(), String> { + let Some((repository, revision, path)) = parse_github_blob_source(source) else { + return if matches!( + evidence, + EvidenceSource::PinnedOracle | EvidenceSource::RealProject + ) { + Err(format!( + "{evidence:?} source must be an immutable GitHub blob URL with a full commit and path" + )) + } else { + Ok(()) + }; + }; + + match evidence { + EvidenceSource::PinnedOracle => { + if repository == meta.upstream_repo && revision == meta.upstream_pin { + Ok(()) + } else { + Err(format!( + "pinned-oracle source must point to {} at the pinned commit", + meta.upstream_repo + )) + } + } + EvidenceSource::RealProject => { + if repository == meta.upstream_repo { + Err( + "real-project source must use its own repository, not the pinned upstream compiler repository" + .into(), + ) + } else if path.is_empty() { + Err("real-project source must identify a repository path".into()) + } else { + Ok(()) + } + } + EvidenceSource::SemanticContract | EvidenceSource::InternalInvariant => Ok(()), + } +} + +fn parse_github_blob_source(source: &str) -> Option<(String, String, String)> { + let rest = source.strip_prefix("https://github.com/")?; + let (repository, rest) = rest.split_once("/blob/")?; + let (revision, path) = rest.split_once('/')?; + let mut repository_parts = repository.split('/'); + let owner = repository_parts.next()?; + let name = repository_parts.next()?; + if owner.is_empty() + || name.is_empty() + || repository_parts.next().is_some() + || revision.len() != 40 + || !revision.bytes().all(|byte| byte.is_ascii_hexdigit()) + || path.is_empty() + || path.contains('?') + || path.contains('#') + { + return None; + } + Some(( + repository.to_string(), + revision.to_string(), + path.to_string(), + )) +} + +fn validate_license(license: &str) -> Result<(), String> { + if license.trim().is_empty() { + Err("// license: must identify the fixture license".into()) + } else { + Ok(()) + } +} + fn infer_evidence(root: &Path, path: &Path, source: &str) -> Option { if path .strip_prefix(root.join("tests/corpus/projects")) @@ -393,3 +474,108 @@ fn expected_matches(expected: ExpectedOutcome, actual: &str) -> bool { | (ExpectedOutcome::HirError, "hir-error") ) } + +#[cfg(test)] +mod tests { + use super::*; + + fn meta() -> matrix::MatrixMeta { + matrix::MatrixMeta { + upstream_repo: "example/ostw".into(), + upstream_pin: "0123456789abcdef0123456789abcdef01234567".into(), + dialect: "ostw".into(), + } + } + + #[test] + fn pinned_evidence_requires_canonical_commit_url() { + let meta = meta(); + assert!( + validate_source( + "https://github.com/example/ostw/blob/0123456789abcdef0123456789abcdef01234567/tests/Parser.cs", + EvidenceSource::PinnedOracle, + &meta, + ) + .is_ok() + ); + assert!( + validate_source( + "https://github.com/example/ostw/blob/deadbeef/tests/Parser.cs", + EvidenceSource::PinnedOracle, + &meta, + ) + .is_err() + ); + } + + #[test] + fn real_project_requires_its_own_immutable_repository() { + let meta = meta(); + assert!( + validate_source( + "https://github.com/example/project/blob/0123456789abcdef0123456789abcdef01234567/src/main.del", + EvidenceSource::RealProject, + &meta, + ) + .is_ok() + ); + assert!( + validate_source( + "https://github.com/example/ostw/blob/0123456789abcdef0123456789abcdef01234567/src/main.del", + EvidenceSource::RealProject, + &meta, + ) + .is_err() + ); + } + + #[test] + fn real_project_rejects_mutable_or_incomplete_source_pointers() { + let meta = meta(); + for source in [ + "https://github.com/example/project/blob/main/src/main.del", + "https://github.com/example/project/blob/0123456789abcdef/src/main.del", + "https://github.com/example/project/blob/0123456789abcdef0123456789abcdef01234567", + ] { + assert!( + validate_source(source, EvidenceSource::RealProject, &meta).is_err(), + "accepted invalid real-project source: {source}" + ); + } + } + + #[test] + fn real_project_metadata_rejects_upstream_compiler_source() { + let root = Path::new("/repo"); + let path = root.join("tests/corpus/projects/real.del"); + let text = "// source: https://github.com/example/ostw/blob/0123456789abcdef0123456789abcdef01234567/src/main.del\n// license: MIT\n// expect: ok\n// evidence: real-project\n"; + let error = parse_metadata( + root, + &path, + text, + &matrix::SupportMatrix { + meta: meta(), + entries: Vec::new(), + }, + ) + .expect_err("upstream compiler identity must not pass as real-project evidence"); + assert!( + error.contains("own repository"), + "unexpected error: {error}" + ); + } + + #[test] + fn license_must_be_present_and_non_empty() { + assert!(validate_license("MIT").is_ok()); + assert!(validate_license(" ").is_err()); + } + + #[test] + fn non_oracle_evidence_can_use_its_own_source_pointer() { + let meta = meta(); + assert!( + validate_source("docs/decisions.md", EvidenceSource::SemanticContract, &meta,).is_ok() + ); + } +} diff --git a/tests/corpus.rs b/tests/corpus.rs index 29ff5e5..e5d0cb2 100644 --- a/tests/corpus.rs +++ b/tests/corpus.rs @@ -211,6 +211,36 @@ fn project_fixtures_load() { errors.join("\n") ); assert!(project.files.len() >= 2, "project {name}: expected imports to load, got files {:?}", project.files.len()); + + let semantic = del_rs::semantic::check_project( + &project, + &del_rs::semantic::provider::NoopProvider::new(), + ); + let semantic_errors: Vec = semantic + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.is_error()) + .map(|diagnostic| format!("{}: {}", diagnostic.code, diagnostic.message)) + .collect(); + assert!( + semantic_errors.is_empty(), + "project {name}: semantic errors:\n{}", + semantic_errors.join("\n") + ); + + let (hir, lower_diagnostics) = del_rs::hir::lower::lower(&semantic); + let mut hir_diagnostics = lower_diagnostics; + hir_diagnostics.extend(del_rs::hir::validate::validate(&hir)); + let hir_errors: Vec = hir_diagnostics + .iter() + .filter(|diagnostic| diagnostic.is_error()) + .map(|diagnostic| format!("{}: {}", diagnostic.code, diagnostic.message)) + .collect(); + assert!( + hir_errors.is_empty(), + "project {name}: HIR errors:\n{}", + hir_errors.join("\n") + ); eprintln!("project {name}: {} files loaded, {} imports", project.files.len(), project.imports.len()); } } @@ -240,11 +270,33 @@ fn compatibility_report_classifies_evidence_and_gaps() { .count() ); assert_eq!(report.summary.unexpected_regressions, 0); + let counted = report.cases.iter().fold([0usize; 5], |mut counts, case| { + let index = match case.status { + del_rs::compatibility::FixtureStatus::Matched => 0, + del_rs::compatibility::FixtureStatus::KnownGap => 1, + del_rs::compatibility::FixtureStatus::Unsupported => 2, + del_rs::compatibility::FixtureStatus::UnexpectedRegression => 3, + del_rs::compatibility::FixtureStatus::Inconclusive => 4, + }; + counts[index] += 1; + counts + }); + assert_eq!( + counted, + [ + report.summary.matched, + report.summary.known_gaps, + report.summary.unsupported, + report.summary.unexpected_regressions, + report.summary.inconclusive, + ] + ); assert!(report.cases.iter().any(|case| { case.fixture.evidence == del_rs::compatibility::EvidenceSource::PinnedOracle })); - assert!(report.cases.iter().any(|case| { - case.fixture.evidence == del_rs::compatibility::EvidenceSource::RealProject + assert!(report.cases.iter().all(|case| { + case.fixture.evidence != del_rs::compatibility::EvidenceSource::RealProject + || !case.fixture.source.contains("ItsDeltin/Overwatch-Script-To-Workshop") })); for case in &report.cases { if case.fixture.expect == del_rs::compatibility::ExpectedOutcome::Unknown { diff --git a/tests/corpus/projects/modules/Container.del b/tests/corpus/projects/modules/Container.del index d8217fe..a9bae25 100644 --- a/tests/corpus/projects/modules/Container.del +++ b/tests/corpus/projects/modules/Container.del @@ -1,6 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Deltinteger/Deltinteger/Modules/Container.del // license: MIT // expect: ok +// evidence: pinned-oracle public void ActivateScoper(in define executor, in define id, define data) { data += id / 100; diff --git a/tests/corpus/projects/modules/Debug Camera.del b/tests/corpus/projects/modules/Debug Camera.del index 4b4f604..77c936a 100644 --- a/tests/corpus/projects/modules/Debug Camera.del +++ b/tests/corpus/projects/modules/Debug Camera.del @@ -1,6 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Deltinteger/Deltinteger/Modules/Debug Camera.del // license: MIT // expect: ok +// evidence: pinned-oracle // note: file name matches the upstream module name; content unchanged define CameraSpeed: IsButtonHeld(EventPlayer(), Button.Ability1) ? 30 : 10; define ToggleKey: IsCommunicating(EventPlayer(), Communication.VoiceLineDown); diff --git a/tests/corpus/projects/modules/Debug Tools.del b/tests/corpus/projects/modules/Debug Tools.del index 62dd18c..c7dab00 100644 --- a/tests/corpus/projects/modules/Debug Tools.del +++ b/tests/corpus/projects/modules/Debug Tools.del @@ -1,6 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Deltinteger/Deltinteger/Modules/Debug Tools.del // license: MIT // expect: ok +// evidence: pinned-oracle // note: file name matches the upstream module name; content unchanged rule: "Debug" { diff --git a/tests/corpus/projects/modules/PathfindEditor.del b/tests/corpus/projects/modules/PathfindEditor.del index c9c1fbf..e4f92b5 100644 --- a/tests/corpus/projects/modules/PathfindEditor.del +++ b/tests/corpus/projects/modules/PathfindEditor.del @@ -1,6 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Deltinteger/Deltinteger/Modules/PathfindEditor.del // license: MIT // expect: ok +// evidence: pinned-oracle // note: pathmap editor; imports '!Container.del' (resolved to the modules dir upstream, to this dir in the corpus) /* diff --git a/tests/corpus/projects/pathfinding/Debug Camera.del b/tests/corpus/projects/pathfinding/Debug Camera.del index 4b4f604..77c936a 100644 --- a/tests/corpus/projects/pathfinding/Debug Camera.del +++ b/tests/corpus/projects/pathfinding/Debug Camera.del @@ -1,6 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Deltinteger/Deltinteger/Modules/Debug Camera.del // license: MIT // expect: ok +// evidence: pinned-oracle // note: file name matches the upstream module name; content unchanged define CameraSpeed: IsButtonHeld(EventPlayer(), Button.Ability1) ? 30 : 10; define ToggleKey: IsCommunicating(EventPlayer(), Communication.VoiceLineDown); diff --git a/tests/corpus/projects/pathfinding/Pathfinding.del b/tests/corpus/projects/pathfinding/Pathfinding.del index 1e685e1..76e4c4a 100644 --- a/tests/corpus/projects/pathfinding/Pathfinding.del +++ b/tests/corpus/projects/pathfinding/Pathfinding.del @@ -1,7 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Examples/Jump-pad pathfinding/Pathfinding.del // license: MIT // expect: ok -// evidence: real-project +// evidence: pinned-oracle // matrix: syntax.imports, syntax.rules, workshop-lowering.events, workshop-lowering.actions, workshop-lowering.values // note: imports '!Debug Camera.del' (resolved to the modules dir upstream, to this dir in the corpus) and 'customGameSettings.json'; uses 'new Pathmap("Map.pathmap")' import "!Debug Camera.del"; diff --git a/tests/corpus/regressions/pathfinding-hud.del b/tests/corpus/regressions/pathfinding-hud.del index a407211..24cc608 100644 --- a/tests/corpus/regressions/pathfinding-hud.del +++ b/tests/corpus/regressions/pathfinding-hud.del @@ -1,7 +1,7 @@ // source: https://github.com/ItsDeltin/Overwatch-Script-To-Workshop/blob/817c1db4bace52123f054ffe10d3d8a06052e687/Examples/Jump-pad%20pathfinding/Pathfinding.del // license: MIT // expect: ok -// evidence: real-project +// evidence: pinned-oracle // matrix: syntax.arguments, syntax.strings.classic-format, workshop-lowering.actions, workshop-lowering.values // note: minimized from the complete Jump-pad pathfinding project; preserves named Workshop arguments and classic formatted text. globalvar define lastDummyBot;