Set up conformance test

.github/workflows/conformance.yml

@@ -0,0 +1,69 @@
+name: Conformance
+
+on:
+  workflow_dispatch:
+    inputs:
+      go_ref:
+        description: "pwn (Go) branch, tag or SHA"
+        default: main
+      julia_ref:
+        description: "PWNModel.jl branch, tag or SHA"
+        default: main
+  pull_request:
+
+jobs:
+  conformance:
+    name: Go vs. Julia
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check out conformance
+        uses: actions/checkout@v6
+        with:
+          path: conformance
+      - name: Check out pwn (Go)
+        uses: actions/checkout@v6
+        with:
+          repository: pwn-model/pwn
+          ref: ${{ inputs.go_ref }}
+          path: pwn
+      - name: Check out PWNModel.jl
+        uses: actions/checkout@v6
+        with:
+          repository: pwn-model/PWNModel.jl
+          ref: ${{ inputs.julia_ref }}
+          path: PWNModel.jl
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.27.x'
+          cache-dependency-path: pwn/go.sum
+      - name: Install Go model dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y libgl1-mesa-dev xorg-dev xvfb
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v3
+        with:
+          version: '1'
+      - uses: julia-actions/cache@v3
+      - name: Install Julia model dependencies
+        run: julia --project=PWNModel.jl -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Go model
+        run: xvfb-run -a conformance/scripts/run_go.sh
+      - name: Run Julia model
+        run: conformance/scripts/run_julia.sh
+      - name: Compare outputs
+        run: python3 conformance/scripts/compare.py
+
+      - name: Upload outputs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conformance-output
+          path: |
+            conformance/out/go
+            conformance/out/julia

.gitignore

@@ -0,0 +1 @@
+/out/

README.md

@@ -1,3 +1,25 @@
 # Model conformance tests
 
-Tests for conformance between the two PWN model implementations.
+Tests for conformance between the two PWN model implementations,
+[pwn](https://github.com/pwn-model/pwn) (Go) and
+[PWNModel.jl](https://github.com/pwn-model/PWNModel.jl) (Julia).
+
+Both implementations run the same headless config
+([`configs/default.yaml`](https://github.com/pwn-model/conformance/blob/main/configs/default.yaml)),
+and their CSV output must be identical.
+
+## Running locally
+
+With `pwn/` and `PWNModel.jl/` checked out next to this repository:
+
+```sh
+scripts/run_go.sh      # runs the Go model with configs/default.yaml     -> out/go/
+scripts/run_julia.sh   # runs the Julia model with configs/default.yaml  -> out/julia/
+scripts/compare.py     # compares the outputs as numbers, non-zero exit on mismatch
+```
+
+## CI
+
+The workflow [conformance.yml](.github/workflows/conformance.yml) is triggered manually
+(Actions → Conformance → Run workflow), with the refs of both model repositories as inputs.
+Model outputs are uploaded as a workflow artifact.

configs/default.yaml

@@ -0,0 +1,93 @@
+seed: 1
+
+resources:
+  - type: WorldSize
+    width: 4000
+    height: 3000
+    cell_size: 10
+    grid_cell_size: 500
+
+systems:
+  # Initialization
+  - type: InitGrids
+
+  - type: InitTrees
+    cell_probability: 0.6
+    tree_probability: 0.9
+    damage_prevalence: 0.03
+    beetle_prevalence: 0.2
+
+  # Systems
+  - type: UpdateTime
+    ticks_per_year: 52
+
+  - type: DiseaseCourse
+    ticks_to_damage: 8
+
+  - type: RandomRelease
+    tick_of_infection: 0
+    num_trees: 10
+    cell_x: 2
+    cell_y: 2
+
+  - type: DamageTrees
+    tick_of_year: 35
+    damage_probability: 0.01
+    removal_probability: 0.333
+
+  - type: BeetleEmergence
+    tick_of_year: 19
+    beetles_per_tree: 5
+    life_expectancy: 5
+
+  - type: TreeAttraction
+    tick_of_year: 18
+    damaged_trees: true
+    half_distance: 50
+    density_radius: 20
+    density_weight: 1.0
+
+  - type: TreeAttraction
+    tick_of_year: 18
+    damaged_trees: false
+    half_distance: 50
+    density_radius: 20
+    density_weight: 1.0
+
+  - type: BeetleMovement
+    steps_per_tick:          7
+    duration_feeding:       6
+    duration_egg_laying:     6
+    leave_tree_probability:  0.25
+    random_walk_probability: 0.6
+
+  - type: BeetleMortality
+
+  - type: Colonization
+    tick_of_year: 20
+    cell_size: 100
+    kernel_half_distance: 50
+    kernel_radius: 300
+    beetles_per_tree: 2.0
+    trees_per_beetle: 1.0
+
+  - type: NematodeInfection
+    infection_probability: 0.01
+
+  # Stop criterion
+  - type: FixedTermination
+    steps: 5200
+
+  # Reporters
+  - type: CSV
+    observer:
+      type: TreePopulation
+    file: tree_pop.csv
+  - type: CSV
+    observer:
+      type: TreeColonization
+    file: tree_col.csv
+  - type: CSV
+    observer:
+      type: TreeDamage
+    file: tree_damage.csv

scripts/compare.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Compares Go and Julia CSV output. Fails on any difference.
+
+Values are compared as numbers, not as strings, as the two implementations
+format them differently (e.g. Go `65138` vs Julia `65138.0`).
+Numbers must be exactly equal, as both implementations are bit-identical.
+
+Usage: scripts/compare.py [go_dir] [julia_dir]   (default: out/go out/julia)
+"""
+
+import csv
+import math
+import sys
+from pathlib import Path
+
+MAX_REPORTED = 10
+
+
+def main():
+    root = Path(__file__).resolve().parent.parent
+    go_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else root / "out" / "go"
+    julia_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else root / "out" / "julia"
+
+    go_files = csv_files(go_dir)
+    julia_files = csv_files(julia_dir)
+
+    ok = True
+    if not go_files:
+        print(f"FAIL no CSV files in {go_dir}")
+        ok = False
+    for f in sorted(go_files - julia_files):
+        print(f"FAIL {f}: only in {go_dir}")
+        ok = False
+    for f in sorted(julia_files - go_files):
+        print(f"FAIL {f}: only in {julia_dir}")
+        ok = False
+
+    for f in sorted(go_files & julia_files):
+        errors = compare_file(go_dir / f, julia_dir / f)
+        if errors:
+            ok = False
+            print(f"FAIL {f}")
+            for e in errors[:MAX_REPORTED]:
+                print(f"     {e}")
+            if len(errors) > MAX_REPORTED:
+                print(f"     ... and {len(errors) - MAX_REPORTED} more")
+        else:
+            print(f"OK   {f}")
+
+    sys.exit(0 if ok else 1)
+
+
+def csv_files(directory):
+    """Returns the paths of all CSV files in directory, relative to it."""
+    return {p.relative_to(directory) for p in directory.rglob("*.csv")}
+
+
+def compare_file(go_file, julia_file):
+    """Returns a list of differences between the two CSV files."""
+    go_rows = read_csv(go_file)
+    julia_rows = read_csv(julia_file)
+
+    if not go_rows or not julia_rows:
+        return ["empty file"]
+    header = go_rows[0]
+    if header != julia_rows[0]:
+        return [f"headers differ: go {go_rows[0]}, julia {julia_rows[0]}"]
+
+    errors = []
+    if len(go_rows) != len(julia_rows):
+        errors.append(f"row counts differ: go {len(go_rows) - 1}, julia {len(julia_rows) - 1}")
+
+    for line, (go_row, julia_row) in enumerate(zip(go_rows[1:], julia_rows[1:]), start=2):
+        if len(go_row) != len(header) or len(julia_row) != len(header):
+            errors.append(f"line {line}: column counts differ from header")
+            continue
+        for col, go_val, julia_val in zip(header, go_row, julia_row):
+            if not equal(go_val, julia_val):
+                errors.append(f"line {line} (t={go_row[0]}), {col}: go {go_val}, julia {julia_val}")
+
+    return errors
+
+
+def read_csv(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))
+
+
+def equal(a, b):
+    """Compares two CSV values as numbers, treating NaN as equal to NaN."""
+    try:
+        x, y = float(a), float(b)
+    except ValueError:
+        return a == b
+    return x == y or (math.isnan(x) and math.isnan(y))
+
+
+if __name__ == "__main__":
+    main()

scripts/run_go.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Runs the Go model with configs/default.yaml. Output goes to out/go/.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+go build -C ../pwn -o "$PWD/pwn" .
+./pwn configs/default.yaml -o out/go

scripts/run_julia.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Runs the Julia model with configs/default.yaml. Output goes to out/julia/.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+julia --project=../PWNModel.jl -e 'using PWNModel; run_model("configs/default.yaml"; out_dir="out/julia")'