From ccb354ee74a8f8959a5540a502a31b0ec70f2f1d Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Sat, 3 Oct 2026 19:08:19 +0000 Subject: [PATCH 1/4] AgentiGrid: keep "};" on MATPOWER cell arrays so ExaGO reads genfuel The MATPOWER parser stored mpc.gentype / mpc.genfuel / mpc.bus_name without the trailing ";", and the writer wrote them back as a bare "}". ExaGO's reader (src/ps/psreaddata.cpp) ends mpc.genfuel at the first line containing "};", so for every case AgentiGrid rewrote, ExaGO ignored the fuel types: all units fell back to the default fuel, ramp rates stayed 0 (SCOPFLOW cannot redispatch in contingencies and reports local infeasibility on ACTIVSg200 at base load, where the original file converges) and wind/solar units stayed at Pmin = Pmax. The parser now keeps "};" / "];"; the writer also terminates raw sections from networks parsed before this fix. tests/grid_files/test_matpower_cell_terminators.py mimics ExaGO's genfuel detection on a round-tripped ACTIVSg200 (3 tests; all fail without the fix). Co-Authored-By: Claude Opus 5.5 --- .../exago_parsers/matpower_parser.py | 8 ++- .../exago_parsers/matpower_writer.py | 15 +++- .../test_matpower_cell_terminators.py | 72 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 agentigrid/tests/grid_files/test_matpower_cell_terminators.py diff --git a/agentigrid/agentigrid/exago_parsers/matpower_parser.py b/agentigrid/agentigrid/exago_parsers/matpower_parser.py index 49576d1d..908bda1e 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_parser.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_parser.py @@ -231,7 +231,11 @@ def parse_matpower(path: Path) -> MATNetwork: if bracket == "{": # Cell array (bus_name, gentype, genfuel, etc.) → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + ("}" if bracket == "{" else "]") + # Keep the MATLAB terminator "};" -- ExaGO's reader finds the end of + # mpc.genfuel by searching for "};", so a bare "}" makes it skip the + # fuel types (and with them the default ramp rates and the + # renewable Pmin=0 relaxation). + raw_text = comment + "\n" + text[match.start():block_start] + block_content + "};" extra_sections[section_name] = raw_text continue @@ -263,7 +267,7 @@ def parse_matpower(path: Path) -> MATNetwork: logger.debug("Parsed %d gencost entries", len(gencost)) else: # Unknown numeric section → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "]" + raw_text = comment + "\n" + text[match.start():block_start] + block_content + "];" extra_sections[section_name] = raw_text logger.debug("Stored unknown section mpc.%s as raw text", section_name) diff --git a/agentigrid/agentigrid/exago_parsers/matpower_writer.py b/agentigrid/agentigrid/exago_parsers/matpower_writer.py index ca787f7f..7ff2898d 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_writer.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_writer.py @@ -64,6 +64,19 @@ def _write_gencost_row(gc: GenCost) -> str: return "\t" + "\t".join(parts) + ";" +def _terminate(raw: str) -> str: + """Ensure a raw section ends with "};" / "];". + + ExaGO locates the end of ``mpc.genfuel`` by the literal "};". Networks + parsed by older versions stored the block without the semicolon; writing + that back made ExaGO ignore the fuel types, leaving every ramp rate at 0 + (SCOPFLOW cannot redispatch in contingencies) and wind/solar fixed at Pmax. + """ + if raw.endswith("}") or raw.endswith("]"): + return raw + ";" + return raw + + def write_matpower(network: MATNetwork, path: Path) -> None: """Write a MATNetwork object to a MATPOWER .m file. @@ -137,7 +150,7 @@ def write_matpower(network: MATNetwork, path: Path) -> None: # --- Extra sections (gentype, genfuel, bus_name, etc.) --- for section_name, raw_text in network.extra_sections.items(): lines.append("") - lines.append(raw_text.rstrip()) + lines.append(_terminate(raw_text.rstrip())) lines.append("") diff --git a/agentigrid/tests/grid_files/test_matpower_cell_terminators.py b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py new file mode 100644 index 00000000..51632824 --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py @@ -0,0 +1,72 @@ +"""Regression: rewritten cases must keep the "};" terminator on cell arrays. + +ExaGO's MATPOWER reader (src/ps/psreaddata.cpp) finds the end of mpc.genfuel +by searching for the literal "};". The writer used to emit a bare "}", so ExaGO +silently ignored the fuel types of every case AgentiGrid rewrote: ramp rates +stayed 0 (SCOPFLOW could not redispatch in contingencies and reported local +infeasibility on ACTIVSg200) and wind/solar units stayed fixed at Pmax. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from agentigrid.exago_parsers.matpower_model import MATNetwork +from agentigrid.exago_parsers.matpower_parser import parse_matpower +from agentigrid.exago_parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + + +def _exago_genfuel_entries(path: Path) -> list[str] | None: + """Mimic psreaddata.cpp: genfuel starts after the 'mpc.genfuel' line and + ends at the first later line containing '};'. None = end never found.""" + lines = path.read_text().splitlines() + start = end = None + for i, line in enumerate(lines): + if start is None and "mpc.genfuel" in line: + start = i + 1 + elif start is not None and "};" in line: + end = i + break + if start is None or end is None: + return None + return [m.group(1) for l in lines[start:end] if (m := re.search(r"'([^']*)'", l))] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_roundtrip_keeps_genfuel_readable_by_exago(tmp_path): + original = _exago_genfuel_entries(ACTIVSG200) + assert original and "wind" in original + out = tmp_path / "rt.m" + write_matpower(parse_matpower(ACTIVSG200), out) + assert _exago_genfuel_entries(out) == original + # every cell array closes with "};", every matrix with "];" + closers = [l.strip() for l in out.read_text().splitlines() if l.strip()[:1] in ("}", "]")] + assert closers and all(c in ("};", "];") for c in closers) + + +def test_writer_terminates_legacy_raw_sections(tmp_path): + """Networks parsed before the fix (e.g. restored sessions) stored '}'.""" + net = MATNetwork(casename="c", version="2", baseMVA=100.0, buses=[], generators=[], + branches=[], gencost=[], header_comments="function mpc = c\n", + extra_sections={"genfuel": "mpc.genfuel = {\n\t'wind';\n}", + "other": "mpc.other = [\n\t1 2;\n]"}) + out = tmp_path / "c.m" + write_matpower(net, out) + text = out.read_text() + assert "\n};\n" in text and "\n];\n" in text + assert _exago_genfuel_entries(out) == ["wind"] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_double_roundtrip_is_stable(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert "};;" not in b.read_text() and "];;" not in b.read_text() + assert _exago_genfuel_entries(b) == _exago_genfuel_entries(ACTIVSG200) From 0189897cf3a7b0d6a99725f63da3516266c860b8 Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Sat, 3 Oct 2026 19:09:45 +0000 Subject: [PATCH 2/4] AgentiGrid tests: OPFLOW fixture with real fuel types; live test skips without opflow tests/fixtures/sample_opflow_output.txt is OPFLOW output for case_ACTIVSg200.m as rewritten by AgentiGrid. It was produced by the writer fixed in the previous commit and therefore listed all 49 generators as COAL; test_gen_189 asserted COAL although the case file says nuclear for bus 189. Regenerated with the fixed writer (25 coal, 17 ng, 6 wind, 1 nuclear) and updated the expected values in tests/results/test_opflow_parser.py; test_fuel_mix_matches_case_file guards against regenerating it with a broken writer again. TestExecutorLive now skips without the opflow binary, like TestMPIGuard. (Develop already stopped the live test from overwriting the fixture in #147.) tests/README.md describes the fixture; claude_code_changes.md has the entry for both commits. Co-Authored-By: Claude Opus 5.5 --- agentigrid/claude_code_changes.md | 24 ++++ agentigrid/tests/README.md | 6 + .../tests/fixtures/sample_opflow_output.txt | 134 +++++++++--------- .../tests/results/test_opflow_parser.py | 16 ++- agentigrid/tests/runner/test_executor.py | 2 + 5 files changed, 110 insertions(+), 72 deletions(-) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 83a815a8..d673d256 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,30 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) + +**Symptom.** SCOPFLOW reported local infeasibility on AgentiGrid-rewritten ACTIVSg200 at base load, while the original file converged. The rewritten file had `mpc.genfuel = { ... }` with a bare `}`. + +**Cause.** `src/ps/psreaddata.cpp` ends `mpc.genfuel` at the first line containing `};`. The parser stored cell arrays (`gentype`, `genfuel`, `bus_name`) without the trailing `;` and the writer wrote them back as `}`, so for every case AgentiGrid rewrote, ExaGO ignored the fuel types: all units took the default fuel, ramp rates were 0, and wind/solar stayed at Pmin = Pmax. + +**Evidence in the repository.** `tests/fixtures/sample_opflow_output.txt` (OPFLOW on AgentiGrid-rewritten ACTIVSg200) listed all 49 generators as COAL; `case_ACTIVSg200.m` has 25 coal, 17 ng, 6 wind and 1 nuclear (bus 189). `test_gen_189` asserted COAL, so it encoded the bug. + +### Changes + +| File | Change | +|---|---| +| `agentigrid/exago_parsers/matpower_parser.py` | Keep `};` / `];` on stored cell-array sections | +| `agentigrid/exago_parsers/matpower_writer.py` | Terminate cell arrays with `};`, including raw sections from networks parsed before the fix | +| `tests/grid_files/test_matpower_cell_terminators.py` | New (3 tests): mimics ExaGO's genfuel detection on a round-tripped ACTIVSg200; all fail without the fix | +| `tests/fixtures/sample_opflow_output.txt` | Regenerated with the fixed writer (Ipopt 3.14.14): real fuel mix | +| `tests/results/test_opflow_parser.py` | Expected values for the new fixture (21 iterations, 0.021 s, bus 189 NUCLEAR); new `test_fuel_mix_matches_case_file` | +| `tests/runner/test_executor.py` | `TestExecutorLive` skips without the `opflow` binary (like `TestMPIGuard`) ; develop (#147) already stopped it from overwriting the fixture | +| `tests/README.md` | What the fixture is and how to keep it consistent when regenerating | + +Fast suite (`python -m pytest -m "not slow"`), without ExaGO binaries: **1083 passed, 13 skipped**. Before this change the same run had 2 failures: `test_solve_time` (the committed fixture no longer matched its expected value) and `test_opflow_run` (no skip without the binary). + +--- + ## C.2/C.3 corrections — metric gating, summary aggregation, predicate audit, sweep dedup (2026-06-23) Four contained fixes from the prompt-13/15/17 runs. No capability behavior changed (capacity numbers, dispatch results, metric/predicate definitions, and the C.1 binding identifier are untouched) — only how results are gated, summarized, audited, and deduplicated. diff --git a/agentigrid/tests/README.md b/agentigrid/tests/README.md index 332826d1..d27c0c02 100644 --- a/agentigrid/tests/README.md +++ b/agentigrid/tests/README.md @@ -17,6 +17,12 @@ Tests are grouped by the part of AgentiGrid they cover: | `e2e/` | End-to-end runs through the agent loop | | `fixtures/` | Shared sample files (e.g. `sample_opflow_output.txt`) | +`fixtures/sample_opflow_output.txt` is OPFLOW output for `case_ACTIVSg200.m` as +rewritten by AgentiGrid; `results/test_opflow_parser.py` checks values from it. +If you regenerate it, write the case with the current MATPOWER writer (so the +fuel types survive) and update the expected values in `results/test_opflow_parser.py`; +`test_fuel_mix_matches_case_file` fails if the fuel mix is lost again. + Run the fast tests from the AgentiGrid root: ```bash diff --git a/agentigrid/tests/fixtures/sample_opflow_output.txt b/agentigrid/tests/fixtures/sample_opflow_output.txt index cf3656c1..2678c865 100644 --- a/agentigrid/tests/fixtures/sample_opflow_output.txt +++ b/agentigrid/tests/fixtures/sample_opflow_output.txt @@ -5,15 +5,15 @@ This program contains Ipopt, a library for large-scale nonlinear optimization. For more information visit https://github.com/coin-or/Ipopt ****************************************************************************** -This is Ipopt version 3.14.20, running with linear solver ma27. +This is Ipopt version 3.14.14, running with linear solver ma27. -Number of nonzeros in equality constraint Jacobian...: 2826 +Number of nonzeros in equality constraint Jacobian...: 2832 Number of nonzeros in inequality constraint Jacobian.: 1958 -Number of nonzeros in Lagrangian Hessian.............: 1956 +Number of nonzeros in Lagrangian Hessian.............: 1992 -Total number of variables............................: 469 +Total number of variables............................: 475 variables with only lower bounds: 0 - variables with lower and upper bounds: 270 + variables with lower and upper bounds: 276 variables with only upper bounds: 0 Total number of equality constraints.................: 400 Total number of inequality constraints...............: 490 @@ -22,52 +22,50 @@ Total number of inequality constraints...............: 490 inequality constraints with only upper bounds: 490 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 0 4.0417248e+04 3.70e+00 4.65e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 - 1 3.9034986e+04 3.40e+00 4.25e+01 -1.0 2.83e+00 - 1.39e-01 8.08e-02f 1 - 2 3.6673124e+04 2.88e+00 3.56e+01 -1.0 7.20e+00 - 4.19e-01 1.48e-01f 1 - 3 3.1578046e+04 1.80e+00 1.23e+02 -1.0 2.11e+01 - 8.80e-01 3.49e-01f 1 - 4 3.0599807e+04 1.52e+00 1.10e+02 -1.0 2.12e+01 - 4.46e-01 1.53e-01f 1 - 5 2.9699291e+04 1.26e+00 9.78e+01 -1.0 1.84e+01 - 4.37e-01 1.71e-01f 1 - 6 2.9309885e+04 1.15e+00 9.04e+01 -1.0 1.59e+01 - 6.85e-01 8.97e-02f 1 - 7 2.8645960e+04 8.46e-01 7.05e+01 -1.0 3.67e+00 - 2.11e-03 2.64e-01f 1 - 8 2.7638522e+04 2.53e-02 7.27e+01 -1.0 3.56e+00 - 9.30e-01 1.00e+00f 1 - 9 2.7640850e+04 8.98e-04 7.46e-01 -1.0 7.87e-01 - 1.00e+00 1.00e+00h 1 + 0 4.0417248e+04 3.70e+00 6.61e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 + 1 3.8873587e+04 3.57e+00 6.36e+01 -1.0 4.46e+00 - 4.92e-02 3.53e-02f 1 + 2 3.6809680e+04 3.38e+00 5.88e+01 -1.0 4.50e+00 - 1.55e-01 5.19e-02f 1 + 3 3.4170482e+04 3.08e+00 4.94e+01 -1.0 3.95e+00 - 2.60e-01 8.73e-02f 1 + 4 3.2530838e+04 2.69e+00 2.99e+01 -1.0 9.39e+00 - 3.74e-01 1.23e-01f 1 + 5 2.9302664e+04 1.78e+00 8.68e+01 -1.0 2.07e+01 - 9.47e-01 3.17e-01f 1 + 6 2.8914676e+04 1.56e+00 7.90e+01 -1.0 1.48e+01 - 3.97e-01 1.26e-01f 1 + 7 2.8108655e+04 6.74e-01 5.46e+01 -1.0 5.25e+00 - 9.65e-01 5.69e-01f 1 + 8 2.7648421e+04 1.63e-02 2.21e+01 -1.0 1.66e+00 - 8.62e-01 1.00e+00f 1 + 9 2.7654216e+04 1.36e-03 1.97e-01 -1.0 3.99e-01 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 10 2.7585786e+04 1.54e-04 9.33e-02 -1.7 3.31e-01 - 1.00e+00 1.00e+00f 1 - 11 2.7569878e+04 1.44e-03 5.82e-02 -2.5 6.72e-02 - 9.88e-01 1.00e+00f 1 - 12 2.7563905e+04 1.32e-02 2.35e-02 -2.5 1.04e-01 - 1.00e+00 1.00e+00h 1 - 13 2.7564551e+04 3.76e-05 1.44e-04 -2.5 2.12e-02 - 1.00e+00 1.00e+00h 1 - 14 2.7558777e+04 6.47e-03 6.79e-02 -3.8 7.82e-02 - 8.63e-01 1.00e+00f 1 - 15 2.7557880e+04 4.68e-03 1.05e-01 -3.8 7.97e-02 - 1.00e+00 3.39e-01h 1 - 16 2.7557754e+04 5.40e-04 4.26e-01 -3.8 1.03e-01 - 1.00e+00 8.88e-01h 1 - 17 2.7557736e+04 9.85e-06 2.13e-04 -3.8 2.55e-02 - 1.00e+00 1.00e+00h 1 - 18 2.7557602e+04 1.37e-05 1.54e-01 -5.7 3.45e-02 - 9.15e-01 7.81e-01h 1 - 19 2.7557572e+04 3.77e-06 1.09e-02 -5.7 9.25e-03 - 1.00e+00 9.65e-01h 1 + 10 2.7574726e+04 1.60e-04 2.22e+00 -2.5 2.75e-01 - 8.77e-01 1.00e+00f 1 + 11 2.7569519e+04 2.59e-03 1.09e-02 -2.5 7.33e-02 - 1.00e+00 1.00e+00f 1 + 12 2.7560217e+04 1.93e-02 1.30e-01 -3.8 1.26e-01 - 7.37e-01 1.00e+00h 1 + 13 2.7557763e+04 1.06e-02 2.11e-02 -3.8 1.04e-01 - 9.63e-01 6.25e-01h 1 + 14 2.7557837e+04 1.74e-03 6.51e-01 -3.8 9.82e-02 - 1.00e+00 8.38e-01h 1 + 15 2.7557760e+04 1.86e-05 4.99e-04 -3.8 2.51e-02 - 1.00e+00 1.00e+00h 1 + 16 2.7557607e+04 1.53e-05 1.57e-01 -5.7 3.40e-02 - 9.19e-01 7.80e-01h 1 + 17 2.7557573e+04 3.73e-06 1.16e-02 -5.7 9.12e-03 - 1.00e+00 9.63e-01h 1 + 18 2.7557573e+04 1.95e-06 9.06e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 + 19 2.7557573e+04 6.41e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 20 2.7557572e+04 1.94e-06 8.78e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 - 21 2.7557572e+04 6.42e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 - 22 2.7557571e+04 2.07e-07 1.75e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 - 23 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 + 20 2.7557571e+04 2.07e-07 1.73e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 + 21 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 -Number of Iterations....: 23 +Number of Iterations....: 21 (scaled) (unscaled) -Objective...............: 1.1868829499149242e+03 2.7557570559622669e+04 -Dual infeasibility......: 2.1624874764514693e-07 5.0209585722739846e-06 -Constraint violation....: 5.1921755382135226e-08 5.1921755382135226e-08 -Variable bound violation: 1.1735202365414921e-08 1.1735202365414921e-08 -Complementarity.........: 2.0827918551588903e-07 4.8359177721495380e-06 -Overall NLP error.......: 2.1624874764514693e-07 5.0209585722739846e-06 +Objective...............: 1.1868829484688854e+03 2.7557570526047901e+04 +Dual infeasibility......: 2.1611411147490891e-07 5.0178325304334832e-06 +Constraint violation....: 5.1889360219194158e-08 5.1889360219194158e-08 +Variable bound violation: 1.2335941157459729e-08 1.2335941157459729e-08 +Complementarity.........: 2.0818450105630093e-07 4.8337193467056586e-06 +Overall NLP error.......: 2.1611411147490891e-07 5.0178325304334832e-06 -Number of objective function evaluations = 25 -Number of objective gradient evaluations = 24 -Number of equality constraint evaluations = 25 -Number of inequality constraint evaluations = 25 -Number of equality constraint Jacobian evaluations = 24 -Number of inequality constraint Jacobian evaluations = 24 -Number of Lagrangian Hessian evaluations = 23 -Total seconds in IPOPT = 0.063 +Number of objective function evaluations = 23 +Number of objective gradient evaluations = 22 +Number of equality constraint evaluations = 23 +Number of inequality constraint evaluations = 23 +Number of equality constraint Jacobian evaluations = 22 +Number of inequality constraint Jacobian evaluations = 22 +Number of Lagrangian Hessian evaluations = 21 +Total seconds in IPOPT = 0.021 EXIT: Optimal Solution Found. ============================================================= @@ -551,7 +549,7 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 51 1 COAL 1.36 0.53 1.36 4.53 -0.55 2.11 52 1 COAL 1.36 0.65 1.36 4.53 -0.55 2.11 53 1 COAL 2.72 0.70 2.72 9.07 -1.11 4.23 -65 1 COAL 86.50 -4.97 86.50 86.50 -21.66 32.04 +65 1 WIND 86.50 -4.97 0.00 86.50 -21.66 32.04 67 1 COAL 1.41 -0.57 1.41 4.70 -0.57 2.19 68 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 69 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 @@ -559,39 +557,39 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 71 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 72 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 73 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 -76 1 COAL 1.20 0.52 1.20 4.00 -0.44 2.04 -77 1 COAL 0.72 0.26 0.72 2.40 -0.27 1.22 -78 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -79 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -90 1 COAL 0.96 -0.36 0.96 3.20 -0.36 1.63 -91 1 COAL 1.50 -0.55 1.50 5.00 -0.55 2.55 -92 0 COAL 0.00 0.00 1.89 6.30 -0.70 3.21 +76 1 NG 1.20 0.52 1.20 4.00 -0.44 2.04 +77 1 NG 0.72 0.26 0.72 2.40 -0.27 1.22 +78 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +79 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +90 1 NG 0.96 -0.36 0.96 3.20 -0.36 1.63 +91 1 NG 1.50 -0.55 1.50 5.00 -0.55 2.55 +92 0 NG 0.00 0.00 1.89 6.30 -0.70 3.21 94 1 COAL 5.40 -2.20 5.40 18.00 -2.20 8.39 -104 1 COAL 67.60 0.80 67.60 67.60 -14.26 21.09 -105 1 COAL 154.80 6.45 154.80 154.80 -28.51 42.17 -114 1 COAL 1.40 0.30 1.40 1.40 -0.24 0.36 -115 1 COAL 133.50 23.61 133.50 133.50 -21.60 31.95 +104 1 WIND 67.60 0.80 0.00 67.60 -14.26 21.09 +105 1 WIND 154.80 6.45 0.00 154.80 -28.51 42.17 +114 1 WIND 1.40 0.30 0.00 1.40 -0.24 0.36 +115 1 WIND 133.50 23.61 0.00 133.50 -21.60 31.95 125 1 COAL 39.02 6.39 39.02 130.05 -15.87 60.60 126 1 COAL 39.02 16.14 39.02 130.05 -15.87 60.60 127 1 COAL 39.02 22.98 39.02 130.05 -15.87 60.60 135 1 COAL 133.92 26.66 133.92 446.40 -54.46 208.02 136 1 COAL 133.92 25.72 133.92 446.40 -54.46 208.02 -147 1 COAL 92.40 11.12 92.40 92.40 -14.47 21.41 +147 1 WIND 92.40 11.12 0.00 92.40 -14.47 21.41 151 1 COAL 1.62 -0.61 1.62 5.40 -0.66 2.52 152 1 COAL 23.17 -5.34 23.17 77.22 -9.42 35.99 153 1 COAL 23.17 -1.87 23.17 77.22 -9.42 35.99 154 1 COAL 23.17 -5.54 23.17 77.22 -9.42 35.99 155 1 COAL 23.17 -5.55 23.17 77.22 -9.42 35.99 -161 0 COAL 0.00 0.00 41.58 138.60 -15.38 70.55 -164 0 COAL 0.00 0.00 3.60 12.00 -1.33 6.11 -165 0 COAL 0.00 0.00 7.80 26.00 -2.89 13.23 -166 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -167 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 -168 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -169 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -170 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 +161 0 NG 0.00 0.00 41.58 138.60 -15.38 70.55 +164 0 NG 0.00 0.00 3.60 12.00 -1.33 6.11 +165 0 NG 0.00 0.00 7.80 26.00 -2.89 13.23 +166 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +167 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 +168 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +169 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +170 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 182 1 COAL 5.25 0.88 5.25 17.50 -2.14 8.16 183 1 COAL 7.98 1.99 7.98 26.60 -3.25 12.40 -189 1 COAL 383.40 -46.67 170.75 569.15 -46.67 209.45 -196 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 -197 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 +189 1 NUCLEAR 383.40 -46.67 170.75 569.15 -46.67 209.45 +196 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 +197 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 diff --git a/agentigrid/tests/results/test_opflow_parser.py b/agentigrid/tests/results/test_opflow_parser.py index 0259a315..50b696fc 100644 --- a/agentigrid/tests/results/test_opflow_parser.py +++ b/agentigrid/tests/results/test_opflow_parser.py @@ -51,10 +51,10 @@ def test_objective_type(self, result: OPFLOWResult): assert result.objective_type == "MIN_GEN_COST" def test_num_iterations(self, result: OPFLOWResult): - assert result.num_iterations == 23 + assert result.num_iterations == 21 def test_solve_time(self, result: OPFLOWResult): - assert result.solve_time == pytest.approx(0.063, abs=0.001) + assert result.solve_time == pytest.approx(0.021, abs=0.001) def test_bus_count(self, result: OPFLOWResult): assert len(result.buses) == 200 @@ -91,10 +91,18 @@ def test_branch_187_189_highest_sf(self, result: OPFLOWResult): assert abs(max_br.Sf - 387.19) < 0.01 def test_gen_189(self, result: OPFLOWResult): - """Gen at bus 189: Pg≈383.40, fuel=COAL.""" + """Gen at bus 189: Pg≈383.40, fuel=NUCLEAR (mpc.genfuel of case_ACTIVSg200.m).""" gen189 = next(g for g in result.generators if g.bus == 189) assert abs(gen189.Pg - 383.40) < 0.01 - assert gen189.fuel == "COAL" + assert gen189.fuel == "NUCLEAR" + + def test_fuel_mix_matches_case_file(self, result: OPFLOWResult): + """All 49 units keep the fuel from mpc.genfuel (25 coal, 17 ng, 6 wind, + 1 nuclear). A fixture regenerated with a writer that drops the "};" + after mpc.genfuel would show every unit as COAL.""" + from collections import Counter + assert Counter(g.fuel for g in result.generators) == { + "COAL": 25, "NG": 17, "WIND": 6, "NUCLEAR": 1} # =========================================================================== diff --git a/agentigrid/tests/runner/test_executor.py b/agentigrid/tests/runner/test_executor.py index 6a201622..366db178 100644 --- a/agentigrid/tests/runner/test_executor.py +++ b/agentigrid/tests/runner/test_executor.py @@ -331,6 +331,8 @@ def test_no_mpirun_for_sopflow_with_mpi_np_1(self, tmp_path: Path): assert cmd_arg[0] != "mpirun" +@pytest.mark.skipif(not _has_opflow, reason="opflow binary not available") +@pytest.mark.skipif(not _has_test_file, reason="case_ACTIVSg200.m not in data/exago/examples/") class TestExecutorLive: def test_opflow_run(self, tmp_path: Path): From 3902dd2418db0e152411bc4197ffab2c71fe37c2 Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Tue, 6 Oct 2026 20:46:20 +0200 Subject: [PATCH 3/4] AgentiGrid MATPOWER reader/writer: reject malformed input, stable round trips Five defects found while specifying the parser and writer: 1. Rows with a non-numeric value were skipped with only a log warning, so a bus, generator or branch could silently disappear; too-short rows crashed with IndexError. Known sections (bus, gen, branch, gencost) now raise ValueError naming the section and row for non-numeric values, column counts outside the section's limits, and non-integral values in integer columns. 2. Valid MATLAB syntax was misread: two rows on one line became one garbage row and comma-separated rows were all dropped. These, '...' continuations, unterminated blocks and a missing mpc.bus now raise ValueError. 3. The writer crashed on Inf/NaN, which the reader accepts. They are written as Inf/-Inf/NaN; bus solution columns no longer lose precision to .4f. 4. Every read/write round trip added another "MATPOWER Case Format" header line. Writing is now idempotent. 5. Case version '1' was accepted although ExaGO reads version 2; it now raises. A missing version still means '2'. CRLF input is normalised on read and output is always LF. The function line always carries network.casename. All 29 case files in datafiles/ still parse and round-trip to equivalent networks. New tests/grid_files/test_matpower_hardening.py (29 tests, 22 fail without the fix). Co-Authored-By: Claude Opus 5.5 --- .../exago_parsers/matpower_parser.py | 132 +++++++--- .../exago_parsers/matpower_writer.py | 39 ++- agentigrid/claude_code_changes.md | 22 ++ .../grid_files/test_matpower_hardening.py | 230 ++++++++++++++++++ 4 files changed, 382 insertions(+), 41 deletions(-) create mode 100644 agentigrid/tests/grid_files/test_matpower_hardening.py diff --git a/agentigrid/agentigrid/exago_parsers/matpower_parser.py b/agentigrid/agentigrid/exago_parsers/matpower_parser.py index 908bda1e..34d0dffd 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_parser.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_parser.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import math import re from pathlib import Path @@ -26,6 +27,25 @@ # Match mpc.
= [ or mpc.
= { _SECTION_RE = re.compile(r"^mpc\.(\w+)\s*=\s*([{\[])", re.MULTILINE) +# Only MATPOWER case format version 2 is supported (ExaGO reads version 2). +_SUPPORTED_VERSIONS = ("2",) + +# Sections parsed into dataclasses; any other section is kept as raw text. +_KNOWN_SECTIONS = ("bus", "gen", "branch", "gencost") +# Column limits per known section. Bus rows have 13 input columns plus up to +# 4 optional solution columns (lam_P, lam_Q, mu_Vmax, mu_Vmin); gen and +# branch rows may carry any number of extra columns, which are kept. +_MIN_COLS = {"bus": 13, "gen": 10, "branch": 13, "gencost": 4} +_MAX_COLS = {"bus": 17} +# 0-based columns that must hold integers (MATPOWER BUS_I, BUS_TYPE, +# BUS_AREA, ZONE; GEN_BUS, GEN_STATUS; F_BUS, T_BUS, BR_STATUS; MODEL, NCOST). +_INT_COLS = { + "bus": (0, 1, 6, 10), + "gen": (0, 7), + "branch": (0, 1, 10), + "gencost": (0, 3), +} + def _parse_float(s: str) -> float: """Parse a string as float, handling MATLAB Inf/NaN.""" @@ -39,21 +59,61 @@ def _parse_float(s: str) -> float: return float(sl) -def _split_data_row(line: str) -> list[str]: - """Split a MATPOWER data row into tokens, stripping comments and semicolons.""" - # Remove inline comment - idx = line.find("%") - if idx >= 0: - line = line[:idx] - # Remove trailing semicolons and whitespace - line = line.rstrip().rstrip(";").rstrip() - return line.split() - +def _parse_rows(section: str, block: str) -> list[list[float]]: + """Parse the rows of a known numeric section (one row per line). -def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, int]: + Anything that would otherwise be misread raises ``ValueError`` naming the + section and the 1-based data row: unsupported syntax (several rows on one + line, commas, ``...`` continuations), non-numeric values, a column count + outside the section's limits, or a non-integral value in an integer column. + Rows are never skipped silently. + """ + rows: list[list[float]] = [] + for line in block.split("\n"): + idx = line.find("%") + if idx >= 0: + line = line[:idx] + line = line.strip().rstrip(";").rstrip() + if not line: + continue + where = f"mpc.{section} row {len(rows) + 1}" + if "..." in line: + raise ValueError(f"{where}: line continuation '...' is not supported") + if ";" in line: + raise ValueError( + f"{where}: several rows on one line are not supported; " + "put each row on its own line" + ) + if "," in line: + raise ValueError( + f"{where}: comma-separated values are not supported; " + "separate values with spaces or tabs" + ) + try: + values = [_parse_float(t) for t in line.split()] + except ValueError: + raise ValueError(f"{where}: non-numeric value in {line!r}") from None + lo, hi = _MIN_COLS[section], _MAX_COLS.get(section) + if len(values) < lo or (hi is not None and len(values) > hi): + expected = f"{lo}-{hi}" if hi is not None else f"at least {lo}" + raise ValueError(f"{where}: {len(values)} columns, expected {expected}") + for col in _INT_COLS[section]: + v = values[col] + if not math.isfinite(v) or v != int(v): + raise ValueError(f"{where}: column {col + 1} must be an integer, got {v!r}") + rows.append(values) + return rows + + +def _extract_section_block( + text: str, start: int, bracket: str, section: str = "?" +) -> tuple[str, int]: """Extract text from *start* until the matching closing bracket. Returns (block_content, end_position). + + Raises: + ValueError: If the block is never closed. """ close = "]" if bracket == "[" else "}" # Find the end of the section @@ -66,6 +126,8 @@ def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, in elif ch == close: depth -= 1 i += 1 + if depth > 0: + raise ValueError(f"mpc.{section}: '{bracket}' block is not terminated") return text[start:i - 1], i @@ -80,6 +142,11 @@ def _collect_section_comment(text: str, section_start: int) -> str: else: break comment_lines.reverse() + # Drop surrounding blank lines so a read/write round trip is stable. + while comment_lines and not comment_lines[0].strip(): + comment_lines.pop(0) + while comment_lines and not comment_lines[-1].strip(): + comment_lines.pop() return "\n".join(comment_lines) @@ -180,7 +247,8 @@ def parse_matpower(path: Path) -> MATNetwork: if not path.exists(): raise FileNotFoundError(f"MATPOWER file not found: {path}") - text = path.read_text(encoding="utf-8") + # Normalise line endings: CRLF (Windows-authored) files read the same as LF. + text = path.read_text(encoding="utf-8").replace("\r\n", "\n") # --- Function name --- m = _FUNC_RE.search(text) @@ -191,6 +259,11 @@ def parse_matpower(path: Path) -> MATNetwork: # --- Version --- m = _VERSION_RE.search(text) version = m.group(1) if m else "2" + if version not in _SUPPORTED_VERSIONS: + raise ValueError( + f"Unsupported MATPOWER case version '{version}' in {path}; " + f"supported: {', '.join(_SUPPORTED_VERSIONS)}" + ) # --- baseMVA --- m = _BASEMVA_RE.search(text) @@ -222,12 +295,13 @@ def parse_matpower(path: Path) -> MATNetwork: if section_name in ("version", "baseMVA"): continue - block_content, _ = _extract_section_block(text, block_start, bracket) + block_content, _ = _extract_section_block(text, block_start, bracket, section_name) # Collect preceding comment line_start = text.rfind("\n", 0, match.start()) + 1 comment = _collect_section_comment(text, line_start) _section_comments[section_name] = comment + prefix = comment + "\n" if comment else "" if bracket == "{": # Cell array (bus_name, gentype, genfuel, etc.) → store as raw @@ -235,23 +309,19 @@ def parse_matpower(path: Path) -> MATNetwork: # mpc.genfuel by searching for "};", so a bare "}" makes it skip the # fuel types (and with them the default ramp rates and the # renewable Pmin=0 relaxation). - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "};" + raw_text = prefix + text[match.start():block_start] + block_content + "};" + extra_sections[section_name] = raw_text + continue + + if section_name not in _KNOWN_SECTIONS: + # Unknown numeric section (areas, dcline, ...) → store as raw + raw_text = prefix + text[match.start():block_start] + block_content + "];" extra_sections[section_name] = raw_text + logger.debug("Stored unknown section mpc.%s as raw text", section_name) continue - # Numeric matrix section — parse rows - rows: list[list[float]] = [] - for line in block_content.split("\n"): - stripped = line.strip() - if not stripped or stripped.startswith("%"): - continue - tokens = _split_data_row(line) - if not tokens: - continue - try: - rows.append([_parse_float(t) for t in tokens]) - except ValueError: - logger.warning("Skipping unparseable row in mpc.%s: %s", section_name, line.strip()) + # Known numeric matrix section — parse rows (raises on bad input) + rows = _parse_rows(section_name, block_content) if section_name == "bus": buses = [_row_to_bus(r) for r in rows] @@ -265,11 +335,9 @@ def parse_matpower(path: Path) -> MATNetwork: elif section_name == "gencost": gencost = [_row_to_gencost(r) for r in rows] logger.debug("Parsed %d gencost entries", len(gencost)) - else: - # Unknown numeric section → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "];" - extra_sections[section_name] = raw_text - logger.debug("Stored unknown section mpc.%s as raw text", section_name) + + if "bus" not in _section_comments: + raise ValueError(f"Cannot find mpc.bus in {path}") # Store section comments on the network for the writer net = MATNetwork( diff --git a/agentigrid/agentigrid/exago_parsers/matpower_writer.py b/agentigrid/agentigrid/exago_parsers/matpower_writer.py index 7ff2898d..9ae60996 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_writer.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_writer.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +import math +import re from pathlib import Path from agentigrid.exago_parsers.matpower_model import ( @@ -15,9 +17,19 @@ logger = logging.getLogger("agentigrid.exago_parsers.matpower") +_FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*\w+", re.MULTILINE) + def _fmt(value: float) -> str: - """Format a numeric value, writing integers without decimals.""" + """Format a numeric value, writing integers without decimals. + + Infinities and NaN are written as MATLAB ``Inf`` / ``-Inf`` / ``NaN`` + (the parser reads them back). + """ + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Inf" if value > 0 else "-Inf" if value == int(value) and abs(value) < 1e15: return str(int(value)) return f"{value:.10g}" @@ -31,7 +43,7 @@ def _write_bus_row(b: Bus) -> str: ] for val in (b.lam_P, b.lam_Q, b.mu_Vmax, b.mu_Vmin): if val is not None: - parts.append(f"{val:.4f}") + parts.append(_fmt(val)) return "\t" + "\t".join(parts) + ";" @@ -93,14 +105,22 @@ def write_matpower(network: MATNetwork, path: Path) -> None: lines: list[str] = [] # --- Header --- - lines.append(network.header_comments.rstrip("\n")) - # Ensure function line is present (header_comments includes it) - if f"function mpc = {network.casename}" not in network.header_comments: - lines.append(f"function mpc = {network.casename}") + # The function line always carries network.casename: an existing one in + # header_comments is renamed, a missing one is added as the first line. + header = network.header_comments.rstrip("\n") + func_line = f"function mpc = {network.casename}" + if _FUNC_RE.search(header): + header = _FUNC_RE.sub(lambda _m: func_line, header, count=1) + else: + header = func_line + ("\n" + header if header else "") + lines.append(header) # --- Version & baseMVA --- - lines.append("") - lines.append(f"%% MATPOWER Case Format : Version {network.version}") + # The header read back from a file already holds the format comment; + # adding it again would grow the file on every read/write round trip. + if "MATPOWER Case Format" not in header: + lines.append("") + lines.append(f"%% MATPOWER Case Format : Version {network.version}") lines.append(f"mpc.version = '{network.version}';") lines.append("") lines.append("%%----- Power Flow Data -----%%") @@ -154,5 +174,6 @@ def write_matpower(network: MATNetwork, path: Path) -> None: lines.append("") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") + # Always LF line endings, whatever the platform or the source file used. + path.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") logger.info("Wrote MATPOWER file: %s", path) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index d673d256..17433ecc 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,28 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## MATPOWER reader/writer hardening — no silent misreads, stable round trips (2026-10-05) + +Found while writing an independent spec for the parser and writer (round-trip, error and syntax rules). Every real case in ExaGO's `datafiles/` (29 case files, `case5` to `case_ACTIVSg10k`, including CRLF ones) still parses and round-trips to an equivalent network; the files are already accepted by the stricter rules. + +| # | Defect | Fix | +|---|---|---| +| 1 | A row with a non-numeric value was skipped with only a log warning, so a bus, generator or branch could silently vanish. Too-short rows crashed with `IndexError`. | `_parse_rows()` raises `ValueError` naming the section and 1-based row for non-numeric values, column counts outside the section's limits (bus 13–17; gen ≥ 10; branch ≥ 13; gencost ≥ 4) and non-integral/Inf/NaN values in integer columns. | +| 2 | Valid MATLAB syntax was misread: two rows on one line became one garbage row; comma-separated rows were all dropped; `...` continuations crashed. | These raise `ValueError` with a clear message. An unterminated `[`/`{` block and a missing `mpc.bus` raise as well. | +| 3 | The writer crashed on `Inf`/`NaN` (`OverflowError`/`ValueError`), which the reader accepts. | Written as `Inf`, `-Inf`, `NaN`. Bus solution columns (`lam_*`, `mu_*`) use the same 10-significant-digit format instead of `.4f`. | +| 4 | Every read/write round trip added another `%% MATPOWER Case Format` line, and blank lines accumulated before raw sections. | The format comment is written only if the header lacks it; collected section comments drop surrounding blank lines. Writing is idempotent. | +| 5 | MATPOWER case version `'1'` was accepted although ExaGO reads version 2. | Raises `ValueError`; a missing version still means `'2'`. | + +Also: CRLF input is normalised on read and the writer always emits LF; the `function mpc = …` line always carries `network.casename` (renamed if the header has another name, added first if missing). Unknown numeric sections (`areas`, `dcline`, …) stay opaque raw text and are not validated. + +| File | Change | +|---|---| +| `agentigrid/exago_parsers/matpower_parser.py` | Fixes 1, 2, 4 (comments), 5; CRLF normalisation | +| `agentigrid/exago_parsers/matpower_writer.py` | Fixes 3, 4 (header); casename/function line; LF output | +| `tests/grid_files/test_matpower_hardening.py` | New (29 tests; 22 fail without the fix) | + +--- + ## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) **Symptom.** SCOPFLOW reported local infeasibility on AgentiGrid-rewritten ACTIVSg200 at base load, while the original file converged. The rewritten file had `mpc.genfuel = { ... }` with a bare `}`. diff --git a/agentigrid/tests/grid_files/test_matpower_hardening.py b/agentigrid/tests/grid_files/test_matpower_hardening.py new file mode 100644 index 00000000..98c0d857 --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_hardening.py @@ -0,0 +1,230 @@ +"""MATPOWER reader/writer hardening: no silent misreads, stable round trips. + +Covers five defects found while specifying the parser: + 1. malformed rows were skipped with only a log warning (a bus could vanish); + 2. valid-but-unsupported MATLAB syntax was misread (two rows on one line + became one garbage row; comma-separated rows were all dropped); + 3. the writer crashed on Inf/NaN, which the reader accepts; + 4. every read/write round trip added another header line; + 5. MATPOWER case version '1' was accepted although ExaGO reads version 2. +""" + +from __future__ import annotations + +import copy +import math +from pathlib import Path + +import pytest + +from agentigrid.exago_parsers.matpower_model import Branch, Bus, GenCost, Generator, MATNetwork +from agentigrid.exago_parsers.matpower_parser import parse_matpower +from agentigrid.exago_parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + +BUS = "\t1\t3\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;\n\t2\t1\t90\t30\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" +GEN = "\t1\t0\t0\t300\t-300\t1\t100\t1\t250\t10;" +BRANCH = "\t1\t2\t0.01\t0.1\t0\t100\t100\t100\t0\t0\t1\t-360\t360;" +GENCOST = "\t2\t0\t0\t3\t0.11\t5\t150;" + + +def _case(*, version="2", bus=BUS, gen=GEN, branch=BRANCH, gencost=GENCOST, extra="", + drop: tuple[str, ...] = ()) -> str: + parts = { + "head": "function mpc = tiny\n%TINY test case\n\n%% MATPOWER Case Format : Version 2\n", + "version": f"mpc.version = '{version}';\n", + "base": "\n%% system MVA base\nmpc.baseMVA = 100;\n", + "bus": f"\n%% bus data\nmpc.bus = [\n{bus}\n];\n", + "gen": f"\n%% generator data\nmpc.gen = [\n{gen}\n];\n", + "branch": f"\n%% branch data\nmpc.branch = [\n{branch}\n];\n", + "gencost": f"\n%% generator cost data\nmpc.gencost = [\n{gencost}\n];\n", + "extra": f"\n{extra}\n" if extra else "", + } + return "".join(v for k, v in parts.items() if k not in drop) + + +def _write(tmp_path: Path, text: str, name: str = "c.m", crlf: bool = False) -> Path: + p = tmp_path / name + data = text.replace("\n", "\r\n") if crlf else text + p.write_bytes(data.encode("utf-8")) + return p + + +# --- 5. version ------------------------------------------------------------ + +def test_version_1_rejected(tmp_path): + with pytest.raises(ValueError, match="version"): + parse_matpower(_write(tmp_path, _case(version="1"))) + + +def test_missing_version_defaults_to_2(tmp_path): + text = _case().replace("mpc.version = '2';\n", "") + assert parse_matpower(_write(tmp_path, text)).version == "2" + + +# --- 1./2. malformed rows and unsupported syntax --------------------------- + +@pytest.mark.parametrize( + "kwargs, section", + [ + ({"bus": BUS.replace("\n", " ")}, "bus"), # two rows, one line + ({"bus": BUS.replace("\t", ",")}, "bus"), # commas + ({"gen": "\t1\t0\t0\t300\t-300 ...\n\t1\t100\t1\t250\t10;"}, "gen"), # continuation + ({"bus": BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;"}, "bus"), # non-numeric + ({"bus": "\t1\t3\t0\t0;"}, "bus"), # too few columns + ({"gen": "\t1\t0\t0;"}, "gen"), + ({"branch": "\t1\t2\t0.01;"}, "branch"), + ({"gencost": "\t2\t0\t0;"}, "gencost"), + ({"bus": BUS + "\n\t3" + "\t1" * 17 + ";"}, "bus"), # 18 bus columns + ({"bus": BUS.replace("\t1\t3\t", "\t1.5\t3\t", 1)}, "bus"), # non-integral bus_i + ({"gen": GEN.replace("\t100\t1\t", "\t100\tInf\t")}, "gen"), # Inf status + ({"branch": BRANCH.replace("\t0\t1\t-360", "\t0\tNaN\t-360")}, "branch"), # NaN status + ], + ids=["rows-one-line", "commas", "continuation", "non-numeric", "short-bus", "short-gen", + "short-branch", "short-gencost", "bus-18-cols", "int-col-1.5", "int-col-inf", + "int-col-nan"], +) +def test_malformed_input_raises_naming_section(tmp_path, kwargs, section): + with pytest.raises(ValueError, match=rf"mpc\.{section}\b"): + parse_matpower(_write(tmp_path, _case(**kwargs))) + + +def test_non_numeric_row_reports_row_number(tmp_path): + bad = BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" + with pytest.raises(ValueError, match=r"mpc\.bus row 3"): + parse_matpower(_write(tmp_path, _case(bus=bad))) + + +def test_unterminated_block_raises(tmp_path): + text = _case(drop=("gencost",)).replace("\n];\n\n%% branch data", "\n\n%% branch data") + with pytest.raises(ValueError, match="terminated"): + parse_matpower(_write(tmp_path, text)) + + +def test_missing_bus_section_raises(tmp_path): + with pytest.raises(ValueError, match="mpc.bus"): + parse_matpower(_write(tmp_path, _case(drop=("bus",)))) + + +def test_missing_gen_and_branch_give_empty_lists(tmp_path): + net = parse_matpower(_write(tmp_path, _case(drop=("gen", "branch", "gencost")))) + assert net.generators == [] and net.branches == [] and net.gencost == [] + + +def test_partial_bus_solution_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5;", 1)))) + b = net.buses[0] + assert b.lam_P == 6.5 and b.lam_Q is None and b.mu_Vmax is None and b.mu_Vmin is None + + +def test_integral_floats_accepted_in_integer_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("\t1\t3\t", "\t1.0\t3.0\t", 1)))) + assert net.buses[0].bus_i == 1 and isinstance(net.buses[0].bus_i, int) + + +# --- line endings ------------------------------------------------------------ + +def test_crlf_reads_like_lf_and_writer_emits_lf(tmp_path): + lf = parse_matpower(_write(tmp_path, _case(), "lf.m")) + crlf = parse_matpower(_write(tmp_path, _case(), "crlf.m", crlf=True)) + assert crlf.buses == lf.buses and crlf.header_comments == lf.header_comments + out = tmp_path / "out.m" + write_matpower(crlf, out) + assert b"\r" not in out.read_bytes() + + +# --- 3. Inf / NaN ---------------------------------------------------------- + +def test_inf_nan_round_trip(tmp_path): + net = parse_matpower(_write(tmp_path, _case())) + net.branches[0].rateA = math.inf + net.generators[0].Qmin = -math.inf + net.buses[1].Pd = math.nan + out = tmp_path / "out.m" + write_matpower(net, out) + back = parse_matpower(out) + assert back.branches[0].rateA == math.inf + assert back.generators[0].Qmin == -math.inf + assert math.isnan(back.buses[1].Pd) + + +def test_solution_columns_keep_precision(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5\t0\t1e-07\t0.000123456789;", 1)))) + out = tmp_path / "out.m" + write_matpower(net, out) + b = parse_matpower(out).buses[0] + assert math.isclose(b.mu_Vmax, 1e-07, rel_tol=1e-9) + assert math.isclose(b.mu_Vmin, 0.000123456789, rel_tol=1e-9) + + +# --- 4. idempotent writing --------------------------------------------------- + +EXTRAS = ( + "%% generator fuel type\nmpc.genfuel = {\n\t'coal';\n};\n\n" + "mpc.areas = [\n\t1\t1;\n];" +) + + +def test_round_trip_text_is_idempotent(tmp_path): + src = _write(tmp_path, _case(extra=EXTRAS), crlf=True) + a, b, c = (tmp_path / n for n in ("a.m", "b.m", "c.m")) + write_matpower(parse_matpower(src), a) + write_matpower(parse_matpower(a), b) + write_matpower(parse_matpower(b), c) + assert a.read_text() == b.read_text() == c.read_text() + assert a.read_text().count("MATPOWER Case Format") == 1 + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_real_case_round_trip_is_idempotent(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert a.read_text() == b.read_text() + + +def test_section_comments_stay_with_their_section(tmp_path): + out = tmp_path / "out.m" + write_matpower(parse_matpower(_write(tmp_path, _case(extra=EXTRAS))), out) + text = out.read_text() + assert text.index("%% generator data") < text.index("mpc.gen =") + assert text.index("];", text.index("mpc.bus =")) < text.index("%% generator data") + assert text.index("%% generator fuel type") < text.index("mpc.genfuel =") + + +# --- casename / function line ------------------------------------------------ + +def _net(header: str, casename: str = "mine") -> MATNetwork: + return MATNetwork( + casename=casename, version="2", baseMVA=100.0, + buses=[Bus(1, 3, 0.0, 0.0, 0.0, 0.0, 1, 1.0, 0.0, 230.0, 1, 1.1, 0.9)], + generators=[Generator(1, 0.0, 0.0, 300.0, -300.0, 1.0, 100.0, 1, 250.0, 10.0)], + branches=[Branch(1, 1, 0.01, 0.1, 0.0, 100.0, 100.0, 100.0, 0.0, 0.0, 1, -360.0, 360.0)], + gencost=[GenCost(2, 0.0, 0.0, 3, [0.11, 5.0, 150.0])], + header_comments=header, + ) + + +def test_casename_wins_over_header_function_line(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net("function mpc = other\n%comment\n", casename="mine"), out) + text = out.read_text() + assert "function mpc = mine" in text and "function mpc = other" not in text + assert text.count("function mpc") == 1 + assert parse_matpower(out).casename == "mine" + + +def test_empty_header_writes_function_line_first(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net(""), out) + assert out.read_text().splitlines()[0] == "function mpc = mine" + assert parse_matpower(out).casename == "mine" + + +def test_writer_does_not_mutate_its_input(tmp_path): + net = parse_matpower(_write(tmp_path, _case(extra=EXTRAS))) + before = copy.deepcopy(net) + write_matpower(net, tmp_path / "out.m") + assert net == before From 9db9ac71e49574aa2532d9f4b7b939fbea0e2950 Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Tue, 6 Oct 2026 20:46:20 +0200 Subject: [PATCH 4/4] AgentiGrid MATPOWER writer/reader: exact near-max floats, idempotent rounding, baseMVA literals Three defects found by property-based round-trip testing: 1. A finite value near the float maximum was written with 10 significant digits that round past it and read back as Inf. The writer now falls back to the exact representation when rounding overflows. 2. A value that rounds to an integer (e.g. 9999999999.5) was written as "1e+10" the first time and "10000000000" after re-reading, so writing was not idempotent. The integer form is now decided on the rounded value. 3. mpc.baseMVA was read with a digits-and-dots pattern: "1e2" (and "1e+15", which the writer itself emits for large values) was silently read as 1, and a negative value was reported as missing. Any numeric literal (sign, exponent, Inf/NaN) on an mpc.baseMVA line is now accepted, text in comments is ignored, and a non-numeric value raises ValueError. 18 more tests in tests/grid_files/test_matpower_hardening.py (12 fail without the fix). All 29 case files in datafiles/ still round-trip. Co-Authored-By: Claude Opus 5.5 --- .../exago_parsers/matpower_parser.py | 9 +++- .../exago_parsers/matpower_writer.py | 14 ++++-- agentigrid/claude_code_changes.md | 8 +++ .../grid_files/test_matpower_hardening.py | 49 +++++++++++++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/agentigrid/agentigrid/exago_parsers/matpower_parser.py b/agentigrid/agentigrid/exago_parsers/matpower_parser.py index 34d0dffd..6520512e 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_parser.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_parser.py @@ -23,7 +23,8 @@ _FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*(\w+)", re.MULTILINE) _VERSION_RE = re.compile(r"mpc\.version\s*=\s*'([^']+)'") -_BASEMVA_RE = re.compile(r"mpc\.baseMVA\s*=\s*([\d.]+)") +# The value is any numeric token (sign, exponent, Inf/NaN) up to ";" or a comment. +_BASEMVA_RE = re.compile(r"^[ \t]*mpc\.baseMVA[ \t]*=[ \t]*([^;%\n]*)", re.MULTILINE) # Match mpc.
= [ or mpc.
= { _SECTION_RE = re.compile(r"^mpc\.(\w+)\s*=\s*([{\[])", re.MULTILINE) @@ -269,7 +270,11 @@ def parse_matpower(path: Path) -> MATNetwork: m = _BASEMVA_RE.search(text) if not m: raise ValueError(f"Cannot find mpc.baseMVA in {path}") - baseMVA = float(m.group(1)) + token = m.group(1).strip() + try: + baseMVA = _parse_float(token) + except ValueError: + raise ValueError(f"mpc.baseMVA: invalid value {token!r} in {path}") from None # --- Header comments (everything before mpc.version or mpc.baseMVA) --- first_mpc = text.find("mpc.") diff --git a/agentigrid/agentigrid/exago_parsers/matpower_writer.py b/agentigrid/agentigrid/exago_parsers/matpower_writer.py index 9ae60996..71db52ff 100644 --- a/agentigrid/agentigrid/exago_parsers/matpower_writer.py +++ b/agentigrid/agentigrid/exago_parsers/matpower_writer.py @@ -30,9 +30,17 @@ def _fmt(value: float) -> str: return "NaN" if math.isinf(value): return "Inf" if value > 0 else "-Inf" - if value == int(value) and abs(value) < 1e15: - return str(int(value)) - return f"{value:.10g}" + text = f"{value:.10g}" + rounded = float(text) + if math.isinf(rounded): + # Rounding to 10 digits pushed a value near the float maximum past + # it; that text would read back as Inf. Write it exactly instead. + text, rounded = repr(value), value + # Decide the integer form on the ROUNDED value, so a value that rounds to + # an integer is written the same way the second time (idempotence). + if rounded == int(rounded) and abs(rounded) < 1e15: + return str(int(rounded)) + return text def _write_bus_row(b: Bus) -> str: diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 17433ecc..523b2651 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -24,6 +24,14 @@ Also: CRLF input is normalised on read and the writer always emits LF; the `func | `agentigrid/exago_parsers/matpower_writer.py` | Fixes 3, 4 (header); casename/function line; LF output | | `tests/grid_files/test_matpower_hardening.py` | New (29 tests; 22 fail without the fix) | +**Follow-up (property-based testing).** Randomised round-trip tests then found three more defects: + +- A finite value near the float maximum was written with 10 significant digits that round past it, and read back as `Inf`. The writer now falls back to the exact representation. +- A value that rounds to an integer (for example `9999999999.5`) was written as `1e+10` the first time and `10000000000` the second, so writing was not idempotent. The integer form is now decided on the rounded value. +- `mpc.baseMVA` was read with a digits-and-dots pattern. `1e2` (or `1e+15`, which the writer itself emits) was silently read as `1`, and negative values were "not found". It now accepts any numeric literal (sign, exponent, `Inf`/`NaN`) on an `mpc.baseMVA` line, ignores the text in comments, and raises `ValueError` for a non-numeric value. + +18 more tests; 12 of them fail without these fixes. + --- ## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) diff --git a/agentigrid/tests/grid_files/test_matpower_hardening.py b/agentigrid/tests/grid_files/test_matpower_hardening.py index 98c0d857..0ffe467f 100644 --- a/agentigrid/tests/grid_files/test_matpower_hardening.py +++ b/agentigrid/tests/grid_files/test_matpower_hardening.py @@ -228,3 +228,52 @@ def test_writer_does_not_mutate_its_input(tmp_path): before = copy.deepcopy(net) write_matpower(net, tmp_path / "out.m") assert net == before + + +# --- number formatting and baseMVA (found by property-based testing) --------- + +@pytest.mark.parametrize( + "value", + [9999999999.5, 123456789012.345, 1e15, 1e16, 0.1 + 0.2, 5e-324, + 1.7976931345e308, -1.7976931348623157e308], + ids=["rounds-to-int", "11-digits", "1e15", "1e16", "0.3", "subnormal", + "near-max", "-max"], +) +def test_number_round_trip_is_close_finite_and_idempotent(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.buses[1].Pd = value + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(net, a) + back = parse_matpower(a) + assert math.isfinite(back.buses[1].Pd) + assert math.isclose(back.buses[1].Pd, value, rel_tol=1e-9, abs_tol=1e-12) + write_matpower(back, b) + assert a.read_text() == b.read_text() + + +@pytest.mark.parametrize( + "text, expected", + [("1e2", 100.0), ("100.0", 100.0), ("-1", -1.0), ("1e+16", 1e16), ("Inf", math.inf)], +) +def test_basemva_accepts_any_numeric_literal(tmp_path, text, expected): + net = parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", f"mpc.baseMVA = {text};"))) + assert net.baseMVA == expected + + +def test_basemva_in_a_comment_is_ignored(tmp_path): + text = _case().replace("%TINY test case\n", "%TINY test case\n% mpc.baseMVA = 5;\n") + assert parse_matpower(_write(tmp_path, text)).baseMVA == 100.0 + + +def test_basemva_non_numeric_raises(tmp_path): + with pytest.raises(ValueError, match="baseMVA"): + parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", "mpc.baseMVA = abc;"))) + + +@pytest.mark.parametrize("value", [1e16, -1.0, 1.7976931345e308]) +def test_basemva_round_trip(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.baseMVA = value + out = tmp_path / "out.m" + write_matpower(net, out) + assert math.isclose(parse_matpower(out).baseMVA, value, rel_tol=1e-9)