From a97248e41452a64643966344bc48e65bd2a1b86a Mon Sep 17 00:00:00 2001 From: mlange-42 Date: Wed, 30 Sep 2026 12:08:43 +0200 Subject: [PATCH 1/5] Set up conformance test --- .github/workflows/conformance.yml | 68 ++++++++++++++++++++++ .gitignore | 1 + README.md | 23 +++++++- configs/default.yaml | 93 +++++++++++++++++++++++++++++++ scripts/compare.sh | 5 ++ scripts/run_go.sh | 6 ++ scripts/run_julia.sh | 5 ++ 7 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 .gitignore create mode 100644 configs/default.yaml create mode 100644 scripts/compare.sh create mode 100644 scripts/run_go.sh create mode 100644 scripts/run_julia.sh diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..3bcfb52 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,68 @@ +name: Conformance + +on: + workflow_dispatch: + inputs: + go_ref: + description: "pwn (Go) branch, tag or SHA" + default: main + julia_ref: + description: "PWNModel.jl branch, tag or SHA" + default: main + +jobs: + conformance: + name: Go vs. Julia + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out conformance + uses: actions/checkout@v6 + with: + path: conformance + - name: Check out pwn (Go) + uses: actions/checkout@v6 + with: + repository: pwn-model/pwn + ref: ${{ inputs.go_ref }} + path: pwn + - name: Check out PWNModel.jl + uses: actions/checkout@v6 + with: + repository: pwn-model/PWNModel.jl + ref: ${{ inputs.julia_ref }} + path: PWNModel.jl + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.27.x' + cache-dependency-path: pwn/go.sum + - name: Install Go model dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y libgl1-mesa-dev xorg-dev xvfb + + - name: Set up Julia + uses: julia-actions/setup-julia@v3 + with: + version: '1' + - uses: julia-actions/cache@v3 + - name: Install Julia model dependencies + run: julia --project=PWNModel.jl -e 'using Pkg; Pkg.instantiate()' + + - name: Run Go model + run: xvfb-run -a conformance/scripts/run_go.sh + - name: Run Julia model + run: conformance/scripts/run_julia.sh + - name: Compare outputs + run: conformance/scripts/compare.sh + + - name: Upload outputs + if: always() + uses: actions/upload-artifact@v4 + with: + name: conformance-output + path: | + conformance/out/go + conformance/out/julia diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a3417b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/out/ diff --git a/README.md b/README.md index 821759f..f246954 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,24 @@ # Model conformance tests -Tests for conformance between the two PWN model implementations. +Tests for conformance between the two PWN model implementations, +[pwn](https://github.com/pwn-model/pwn) (Go) and +[PWNModel.jl](https://github.com/pwn-model/PWNModel.jl) (Julia). + +Both implementations run the same headless config (`configs/default.yaml`), +and their CSV output must be identical. + +## Running locally + +With `pwn/` and `PWNModel.jl/` checked out next to this repository: + +```sh +scripts/run_go.sh # runs the Go model with configs/default.yaml -> out/go/ +scripts/run_julia.sh # runs the Julia model with configs/default.yaml -> out/julia/ +scripts/compare.sh # compares the outputs, non-zero exit on mismatch +``` + +## CI + +The workflow [conformance.yml](.github/workflows/conformance.yml) is triggered manually +(Actions → Conformance → Run workflow), with the refs of both model repositories as inputs. +Model outputs are uploaded as a workflow artifact. diff --git a/configs/default.yaml b/configs/default.yaml new file mode 100644 index 0000000..ea71bd5 --- /dev/null +++ b/configs/default.yaml @@ -0,0 +1,93 @@ +seed: 1 + +resources: + - type: WorldSize + width: 4000 + height: 3000 + cell_size: 10 + grid_cell_size: 500 + +systems: + # Initialization + - type: InitGrids + + - type: InitTrees + cell_probability: 0.6 + tree_probability: 0.9 + damage_prevalence: 0.03 + beetle_prevalence: 0.2 + + # Systems + - type: UpdateTime + ticks_per_year: 52 + + - type: DiseaseCourse + ticks_to_damage: 8 + + - type: RandomRelease + tick_of_infection: 0 + num_trees: 10 + cell_x: 2 + cell_y: 2 + + - type: DamageTrees + tick_of_year: 35 + damage_probability: 0.01 + removal_probability: 0.333 + + - type: BeetleEmergence + tick_of_year: 19 + beetles_per_tree: 5 + life_expectancy: 5 + + - type: TreeAttraction + tick_of_year: 18 + damaged_trees: true + half_distance: 50 + density_radius: 20 + density_weight: 1.0 + + - type: TreeAttraction + tick_of_year: 18 + damaged_trees: false + half_distance: 50 + density_radius: 20 + density_weight: 1.0 + + - type: BeetleMovement + steps_per_tick: 7 + duration_feeding: 6 + duration_egg_laying: 6 + leave_tree_probability: 0.25 + random_walk_probability: 0.6 + + - type: BeetleMortality + + - type: Colonization + tick_of_year: 20 + cell_size: 100 + kernel_half_distance: 50 + kernel_radius: 300 + beetles_per_tree: 2.0 + trees_per_beetle: 1.0 + + - type: NematodeInfection + infection_probability: 0.01 + + # Stop criterion + - type: FixedTermination + steps: 5200 + + # Reporters + - type: CSV + observer: + type: TreePopulation + file: tree_pop.csv + - type: CSV + observer: + type: TreeColonization + file: tree_col.csv + - type: CSV + observer: + type: TreeDamage + file: tree_damage.csv diff --git a/scripts/compare.sh b/scripts/compare.sh new file mode 100644 index 0000000..46cd080 --- /dev/null +++ b/scripts/compare.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Compares Go and Julia output. Fails on any difference. +set -euo pipefail +cd "$(dirname "$0")/.." +diff -r out/go out/julia diff --git a/scripts/run_go.sh b/scripts/run_go.sh new file mode 100644 index 0000000..04c70aa --- /dev/null +++ b/scripts/run_go.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Runs the Go model with configs/default.yaml. Output goes to out/go/. +set -euo pipefail +cd "$(dirname "$0")/.." +go build -C ../pwn -o "$PWD/pwn" . +./pwn configs/default.yaml -o out/go diff --git a/scripts/run_julia.sh b/scripts/run_julia.sh new file mode 100644 index 0000000..9547c47 --- /dev/null +++ b/scripts/run_julia.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Runs the Julia model with configs/default.yaml. Output goes to out/julia/. +set -euo pipefail +cd "$(dirname "$0")/.." +julia --project=../PWNModel.jl -e 'using PWNModel; run_model("configs/default.yaml"; out_dir="out/julia")' From 916aa9b2d61db0a6e7fd2eb23775a3a757e8b100 Mon Sep 17 00:00:00 2001 From: mlange-42 Date: Wed, 30 Sep 2026 12:12:51 +0200 Subject: [PATCH 2/5] Run CI on PRs --- .github/workflows/conformance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 3bcfb52..7674ba4 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -9,6 +9,7 @@ on: julia_ref: description: "PWNModel.jl branch, tag or SHA" default: main + pull_request: jobs: conformance: From 2175c9dc31f972456fcc31aad233ae27ff3d3a05 Mon Sep 17 00:00:00 2001 From: mlange-42 Date: Wed, 30 Sep 2026 12:17:50 +0200 Subject: [PATCH 3/5] fix file modes --- scripts/compare.sh | 0 scripts/run_go.sh | 0 scripts/run_julia.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/compare.sh mode change 100644 => 100755 scripts/run_go.sh mode change 100644 => 100755 scripts/run_julia.sh diff --git a/scripts/compare.sh b/scripts/compare.sh old mode 100644 new mode 100755 diff --git a/scripts/run_go.sh b/scripts/run_go.sh old mode 100644 new mode 100755 diff --git a/scripts/run_julia.sh b/scripts/run_julia.sh old mode 100644 new mode 100755 From 937d0fc9b69d9abe98d4d7b654b200f965b94d11 Mon Sep 17 00:00:00 2001 From: mlange-42 Date: Wed, 30 Sep 2026 13:17:51 +0200 Subject: [PATCH 4/5] use python to compare results --- .github/workflows/conformance.yml | 2 +- README.md | 2 +- scripts/compare.py | 99 +++++++++++++++++++++++++++++++ scripts/compare.sh | 5 -- 4 files changed, 101 insertions(+), 7 deletions(-) create mode 100755 scripts/compare.py delete mode 100755 scripts/compare.sh diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 7674ba4..cdd17d1 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -57,7 +57,7 @@ jobs: - name: Run Julia model run: conformance/scripts/run_julia.sh - name: Compare outputs - run: conformance/scripts/compare.sh + run: python3 conformance/scripts/compare.py - name: Upload outputs if: always() diff --git a/README.md b/README.md index f246954..6012887 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ With `pwn/` and `PWNModel.jl/` checked out next to this repository: ```sh scripts/run_go.sh # runs the Go model with configs/default.yaml -> out/go/ scripts/run_julia.sh # runs the Julia model with configs/default.yaml -> out/julia/ -scripts/compare.sh # compares the outputs, non-zero exit on mismatch +scripts/compare.py # compares the outputs as numbers, non-zero exit on mismatch ``` ## CI diff --git a/scripts/compare.py b/scripts/compare.py new file mode 100755 index 0000000..e7110a3 --- /dev/null +++ b/scripts/compare.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Compares Go and Julia CSV output. Fails on any difference. + +Values are compared as numbers, not as strings, as the two implementations +format them differently (e.g. Go `65138` vs Julia `65138.0`). +Numbers must be exactly equal, as both implementations are bit-identical. + +Usage: scripts/compare.py [go_dir] [julia_dir] (default: out/go out/julia) +""" + +import csv +import math +import sys +from pathlib import Path + +MAX_REPORTED = 10 + + +def main(): + root = Path(__file__).resolve().parent.parent + go_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else root / "out" / "go" + julia_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else root / "out" / "julia" + + go_files = csv_files(go_dir) + julia_files = csv_files(julia_dir) + + ok = True + if not go_files: + print(f"FAIL no CSV files in {go_dir}") + ok = False + for f in sorted(go_files - julia_files): + print(f"FAIL {f}: only in {go_dir}") + ok = False + for f in sorted(julia_files - go_files): + print(f"FAIL {f}: only in {julia_dir}") + ok = False + + for f in sorted(go_files & julia_files): + errors = compare_file(go_dir / f, julia_dir / f) + if errors: + ok = False + print(f"FAIL {f}") + for e in errors[:MAX_REPORTED]: + print(f" {e}") + if len(errors) > MAX_REPORTED: + print(f" ... and {len(errors) - MAX_REPORTED} more") + else: + print(f"OK {f}") + + sys.exit(0 if ok else 1) + + +def csv_files(directory): + """Returns the paths of all CSV files in directory, relative to it.""" + return {p.relative_to(directory) for p in directory.rglob("*.csv")} + + +def compare_file(go_file, julia_file): + """Returns a list of differences between the two CSV files.""" + go_rows = read_csv(go_file) + julia_rows = read_csv(julia_file) + + if not go_rows or not julia_rows: + return ["empty file"] + header = go_rows[0] + if header != julia_rows[0]: + return [f"headers differ: go {go_rows[0]}, julia {julia_rows[0]}"] + + errors = [] + if len(go_rows) != len(julia_rows): + errors.append(f"row counts differ: go {len(go_rows) - 1}, julia {len(julia_rows) - 1}") + + for line, (go_row, julia_row) in enumerate(zip(go_rows[1:], julia_rows[1:]), start=2): + if len(go_row) != len(header) or len(julia_row) != len(header): + errors.append(f"line {line}: column counts differ from header") + continue + for col, go_val, julia_val in zip(header, go_row, julia_row): + if not equal(go_val, julia_val): + errors.append(f"line {line} (t={go_row[0]}), {col}: go {go_val}, julia {julia_val}") + + return errors + + +def read_csv(path): + with open(path, newline="") as f: + return list(csv.reader(f)) + + +def equal(a, b): + """Compares two CSV values as numbers, treating NaN as equal to NaN.""" + try: + x, y = float(a), float(b) + except ValueError: + return a == b + return x == y or (math.isnan(x) and math.isnan(y)) + + +if __name__ == "__main__": + main() diff --git a/scripts/compare.sh b/scripts/compare.sh deleted file mode 100755 index 46cd080..0000000 --- a/scripts/compare.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Compares Go and Julia output. Fails on any difference. -set -euo pipefail -cd "$(dirname "$0")/.." -diff -r out/go out/julia From 5be291b29a9c664dc10e966382c3f1f987c66131 Mon Sep 17 00:00:00 2001 From: mlange-42 Date: Wed, 30 Sep 2026 20:37:50 +0200 Subject: [PATCH 5/5] link config in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6012887..6f46958 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Tests for conformance between the two PWN model implementations, [pwn](https://github.com/pwn-model/pwn) (Go) and [PWNModel.jl](https://github.com/pwn-model/PWNModel.jl) (Julia). -Both implementations run the same headless config (`configs/default.yaml`), +Both implementations run the same headless config +([`configs/default.yaml`](https://github.com/pwn-model/conformance/blob/main/configs/default.yaml)), and their CSV output must be identical. ## Running locally