From 368cd6a10a8038931381661fac755a3218e4ec0d Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Sat, 3 Oct 2026 19:08:19 +0000 Subject: [PATCH 1/5] AgentiGrid: keep "};" on MATPOWER cell arrays so ExaGO reads genfuel The MATPOWER parser stored mpc.gentype / mpc.genfuel / mpc.bus_name without the trailing ";", and the writer wrote them back as a bare "}". ExaGO's reader (src/ps/psreaddata.cpp) ends mpc.genfuel at the first line containing "};", so for every case AgentiGrid rewrote, ExaGO ignored the fuel types: all units fell back to the default fuel, ramp rates stayed 0 (SCOPFLOW cannot redispatch in contingencies and reports local infeasibility on ACTIVSg200 at base load, where the original file converges) and wind/solar units stayed at Pmin = Pmax. The parser now keeps "};" / "];"; the writer also terminates raw sections from networks parsed before this fix. tests/grid_files/test_matpower_cell_terminators.py mimics ExaGO's genfuel detection on a round-tripped ACTIVSg200 (3 tests; all fail without the fix). Co-Authored-By: Claude Opus 5.5 --- .../agentigrid/parsers/matpower_parser.py | 8 ++- .../agentigrid/parsers/matpower_writer.py | 15 +++- .../test_matpower_cell_terminators.py | 72 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 agentigrid/tests/grid_files/test_matpower_cell_terminators.py diff --git a/agentigrid/agentigrid/parsers/matpower_parser.py b/agentigrid/agentigrid/parsers/matpower_parser.py index dc381b0b..ff50a626 100644 --- a/agentigrid/agentigrid/parsers/matpower_parser.py +++ b/agentigrid/agentigrid/parsers/matpower_parser.py @@ -231,7 +231,11 @@ def parse_matpower(path: Path) -> MATNetwork: if bracket == "{": # Cell array (bus_name, gentype, genfuel, etc.) → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + ("}" if bracket == "{" else "]") + # Keep the MATLAB terminator "};" -- ExaGO's reader finds the end of + # mpc.genfuel by searching for "};", so a bare "}" makes it skip the + # fuel types (and with them the default ramp rates and the + # renewable Pmin=0 relaxation). + raw_text = comment + "\n" + text[match.start():block_start] + block_content + "};" extra_sections[section_name] = raw_text continue @@ -263,7 +267,7 @@ def parse_matpower(path: Path) -> MATNetwork: logger.debug("Parsed %d gencost entries", len(gencost)) else: # Unknown numeric section → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "]" + raw_text = comment + "\n" + text[match.start():block_start] + block_content + "];" extra_sections[section_name] = raw_text logger.debug("Stored unknown section mpc.%s as raw text", section_name) diff --git a/agentigrid/agentigrid/parsers/matpower_writer.py b/agentigrid/agentigrid/parsers/matpower_writer.py index 10bef308..181ce579 100644 --- a/agentigrid/agentigrid/parsers/matpower_writer.py +++ b/agentigrid/agentigrid/parsers/matpower_writer.py @@ -64,6 +64,19 @@ def _write_gencost_row(gc: GenCost) -> str: return "\t" + "\t".join(parts) + ";" +def _terminate(raw: str) -> str: + """Ensure a raw section ends with "};" / "];". + + ExaGO locates the end of ``mpc.genfuel`` by the literal "};". Networks + parsed by older versions stored the block without the semicolon; writing + that back made ExaGO ignore the fuel types, leaving every ramp rate at 0 + (SCOPFLOW cannot redispatch in contingencies) and wind/solar fixed at Pmax. + """ + if raw.endswith("}") or raw.endswith("]"): + return raw + ";" + return raw + + def write_matpower(network: MATNetwork, path: Path) -> None: """Write a MATNetwork object to a MATPOWER .m file. @@ -137,7 +150,7 @@ def write_matpower(network: MATNetwork, path: Path) -> None: # --- Extra sections (gentype, genfuel, bus_name, etc.) --- for section_name, raw_text in network.extra_sections.items(): lines.append("") - lines.append(raw_text.rstrip()) + lines.append(_terminate(raw_text.rstrip())) lines.append("") diff --git a/agentigrid/tests/grid_files/test_matpower_cell_terminators.py b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py new file mode 100644 index 00000000..f845cf17 --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py @@ -0,0 +1,72 @@ +"""Regression: rewritten cases must keep the "};" terminator on cell arrays. + +ExaGO's MATPOWER reader (src/ps/psreaddata.cpp) finds the end of mpc.genfuel +by searching for the literal "};". The writer used to emit a bare "}", so ExaGO +silently ignored the fuel types of every case AgentiGrid rewrote: ramp rates +stayed 0 (SCOPFLOW could not redispatch in contingencies and reported local +infeasibility on ACTIVSg200) and wind/solar units stayed fixed at Pmax. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from agentigrid.parsers.matpower_model import MATNetwork +from agentigrid.parsers.matpower_parser import parse_matpower +from agentigrid.parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + + +def _exago_genfuel_entries(path: Path) -> list[str] | None: + """Mimic psreaddata.cpp: genfuel starts after the 'mpc.genfuel' line and + ends at the first later line containing '};'. None = end never found.""" + lines = path.read_text().splitlines() + start = end = None + for i, line in enumerate(lines): + if start is None and "mpc.genfuel" in line: + start = i + 1 + elif start is not None and "};" in line: + end = i + break + if start is None or end is None: + return None + return [m.group(1) for l in lines[start:end] if (m := re.search(r"'([^']*)'", l))] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_roundtrip_keeps_genfuel_readable_by_exago(tmp_path): + original = _exago_genfuel_entries(ACTIVSG200) + assert original and "wind" in original + out = tmp_path / "rt.m" + write_matpower(parse_matpower(ACTIVSG200), out) + assert _exago_genfuel_entries(out) == original + # every cell array closes with "};", every matrix with "];" + closers = [l.strip() for l in out.read_text().splitlines() if l.strip()[:1] in ("}", "]")] + assert closers and all(c in ("};", "];") for c in closers) + + +def test_writer_terminates_legacy_raw_sections(tmp_path): + """Networks parsed before the fix (e.g. restored sessions) stored '}'.""" + net = MATNetwork(casename="c", version="2", baseMVA=100.0, buses=[], generators=[], + branches=[], gencost=[], header_comments="function mpc = c\n", + extra_sections={"genfuel": "mpc.genfuel = {\n\t'wind';\n}", + "other": "mpc.other = [\n\t1 2;\n]"}) + out = tmp_path / "c.m" + write_matpower(net, out) + text = out.read_text() + assert "\n};\n" in text and "\n];\n" in text + assert _exago_genfuel_entries(out) == ["wind"] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_double_roundtrip_is_stable(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert "};;" not in b.read_text() and "];;" not in b.read_text() + assert _exago_genfuel_entries(b) == _exago_genfuel_entries(ACTIVSG200) From 67f7d6f9f56b44a11b67ea2bb62ed9538962f2f4 Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Sat, 3 Oct 2026 19:09:45 +0000 Subject: [PATCH 2/5] AgentiGrid tests: OPFLOW fixture with real fuel types; live test no longer overwrites it tests/fixtures/sample_opflow_output.txt is OPFLOW output for case_ACTIVSg200.m as rewritten by AgentiGrid. It was produced by the writer fixed in the previous commit and therefore listed all 49 generators as COAL; test_gen_189 asserted COAL although the case file says nuclear for bus 189. Regenerated with the fixed writer (25 coal, 17 ng, 6 wind, 1 nuclear) and updated the expected values in tests/results/test_opflow_parser.py; test_fuel_mix_matches_case_file guards against regenerating it with a broken writer again. TestExecutorLive::test_opflow_run overwrote this tracked fixture on every run with an opflow binary, so the parser tests depended on the machine and test order (test_solve_time already failed against the committed fixture). It now writes the fixture only with AGENTIGRID_REGENERATE_FIXTURES=1, and the class skips without the binary like TestMPIGuard. tests/README.md documents the regeneration; claude_code_changes.md has the entry for both commits. Fast suite without ExaGO binaries: 1083 passed, 13 skipped (before: 2 failed). Co-Authored-By: Claude Opus 5.5 --- agentigrid/claude_code_changes.md | 24 ++++ agentigrid/tests/README.md | 6 + .../tests/fixtures/sample_opflow_output.txt | 134 +++++++++--------- .../tests/results/test_opflow_parser.py | 16 ++- agentigrid/tests/runner/test_executor.py | 10 +- 5 files changed, 116 insertions(+), 74 deletions(-) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 5c83fa3a..7a4bece1 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,30 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) + +**Symptom.** SCOPFLOW reported local infeasibility on AgentiGrid-rewritten ACTIVSg200 at base load, while the original file converged. The rewritten file had `mpc.genfuel = { ... }` with a bare `}`. + +**Cause.** `src/ps/psreaddata.cpp` ends `mpc.genfuel` at the first line containing `};`. The parser stored cell arrays (`gentype`, `genfuel`, `bus_name`) without the trailing `;` and the writer wrote them back as `}`, so for every case AgentiGrid rewrote, ExaGO ignored the fuel types: all units took the default fuel, ramp rates were 0, and wind/solar stayed at Pmin = Pmax. + +**Evidence in the repository.** `tests/fixtures/sample_opflow_output.txt` (OPFLOW on AgentiGrid-rewritten ACTIVSg200) listed all 49 generators as COAL; `case_ACTIVSg200.m` has 25 coal, 17 ng, 6 wind and 1 nuclear (bus 189). `test_gen_189` asserted COAL, so it encoded the bug. + +### Changes + +| File | Change | +|---|---| +| `agentigrid/parsers/matpower_parser.py` | Keep `};` / `];` on stored cell-array sections | +| `agentigrid/parsers/matpower_writer.py` | Terminate cell arrays with `};`, including raw sections from networks parsed before the fix | +| `tests/grid_files/test_matpower_cell_terminators.py` | New (3 tests): mimics ExaGO's genfuel detection on a round-tripped ACTIVSg200; all fail without the fix | +| `tests/fixtures/sample_opflow_output.txt` | Regenerated with the fixed writer (Ipopt 3.14.14): real fuel mix | +| `tests/results/test_opflow_parser.py` | Expected values for the new fixture (21 iterations, 0.021 s, bus 189 NUCLEAR); new `test_fuel_mix_matches_case_file` | +| `tests/runner/test_executor.py` | `TestExecutorLive` skips without the `opflow` binary (like `TestMPIGuard`) and no longer overwrites the tracked fixture on every run; regenerate with `AGENTIGRID_REGENERATE_FIXTURES=1` | +| `tests/README.md` | How to regenerate the fixture | + +Fast suite (`python -m pytest -m "not slow"`), without ExaGO binaries: **1083 passed, 13 skipped**. Before this change the same run had 2 failures: `test_solve_time` (the committed fixture no longer matched its expected value) and `test_opflow_run` (no skip without the binary). + +--- + ## C.2/C.3 corrections — metric gating, summary aggregation, predicate audit, sweep dedup (2026-06-23) Four contained fixes from the prompt-13/15/17 runs. No capability behavior changed (capacity numbers, dispatch results, metric/predicate definitions, and the C.1 binding identifier are untouched) — only how results are gated, summarized, audited, and deduplicated. diff --git a/agentigrid/tests/README.md b/agentigrid/tests/README.md index d9aec734..261752ae 100644 --- a/agentigrid/tests/README.md +++ b/agentigrid/tests/README.md @@ -16,6 +16,12 @@ Tests are grouped by the part of AgentiGrid they cover: | `e2e/` | End-to-end runs through the agent loop | | `fixtures/` | Shared sample files (e.g. `sample_opflow_output.txt`) | +`fixtures/sample_opflow_output.txt` is OPFLOW output for `case_ACTIVSg200.m` as +rewritten by AgentiGrid; `results/test_opflow_parser.py` checks values from it. +To regenerate it (needs the `opflow` binary), run the live executor test with +`AGENTIGRID_REGENERATE_FIXTURES=1 python -m pytest tests/runner/test_executor.py -k test_opflow_run` +and update the expected values in `results/test_opflow_parser.py`. + Run the fast tests from the AgentiGrid root: ```bash diff --git a/agentigrid/tests/fixtures/sample_opflow_output.txt b/agentigrid/tests/fixtures/sample_opflow_output.txt index cf3656c1..2678c865 100644 --- a/agentigrid/tests/fixtures/sample_opflow_output.txt +++ b/agentigrid/tests/fixtures/sample_opflow_output.txt @@ -5,15 +5,15 @@ This program contains Ipopt, a library for large-scale nonlinear optimization. For more information visit https://github.com/coin-or/Ipopt ****************************************************************************** -This is Ipopt version 3.14.20, running with linear solver ma27. +This is Ipopt version 3.14.14, running with linear solver ma27. -Number of nonzeros in equality constraint Jacobian...: 2826 +Number of nonzeros in equality constraint Jacobian...: 2832 Number of nonzeros in inequality constraint Jacobian.: 1958 -Number of nonzeros in Lagrangian Hessian.............: 1956 +Number of nonzeros in Lagrangian Hessian.............: 1992 -Total number of variables............................: 469 +Total number of variables............................: 475 variables with only lower bounds: 0 - variables with lower and upper bounds: 270 + variables with lower and upper bounds: 276 variables with only upper bounds: 0 Total number of equality constraints.................: 400 Total number of inequality constraints...............: 490 @@ -22,52 +22,50 @@ Total number of inequality constraints...............: 490 inequality constraints with only upper bounds: 490 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 0 4.0417248e+04 3.70e+00 4.65e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 - 1 3.9034986e+04 3.40e+00 4.25e+01 -1.0 2.83e+00 - 1.39e-01 8.08e-02f 1 - 2 3.6673124e+04 2.88e+00 3.56e+01 -1.0 7.20e+00 - 4.19e-01 1.48e-01f 1 - 3 3.1578046e+04 1.80e+00 1.23e+02 -1.0 2.11e+01 - 8.80e-01 3.49e-01f 1 - 4 3.0599807e+04 1.52e+00 1.10e+02 -1.0 2.12e+01 - 4.46e-01 1.53e-01f 1 - 5 2.9699291e+04 1.26e+00 9.78e+01 -1.0 1.84e+01 - 4.37e-01 1.71e-01f 1 - 6 2.9309885e+04 1.15e+00 9.04e+01 -1.0 1.59e+01 - 6.85e-01 8.97e-02f 1 - 7 2.8645960e+04 8.46e-01 7.05e+01 -1.0 3.67e+00 - 2.11e-03 2.64e-01f 1 - 8 2.7638522e+04 2.53e-02 7.27e+01 -1.0 3.56e+00 - 9.30e-01 1.00e+00f 1 - 9 2.7640850e+04 8.98e-04 7.46e-01 -1.0 7.87e-01 - 1.00e+00 1.00e+00h 1 + 0 4.0417248e+04 3.70e+00 6.61e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 + 1 3.8873587e+04 3.57e+00 6.36e+01 -1.0 4.46e+00 - 4.92e-02 3.53e-02f 1 + 2 3.6809680e+04 3.38e+00 5.88e+01 -1.0 4.50e+00 - 1.55e-01 5.19e-02f 1 + 3 3.4170482e+04 3.08e+00 4.94e+01 -1.0 3.95e+00 - 2.60e-01 8.73e-02f 1 + 4 3.2530838e+04 2.69e+00 2.99e+01 -1.0 9.39e+00 - 3.74e-01 1.23e-01f 1 + 5 2.9302664e+04 1.78e+00 8.68e+01 -1.0 2.07e+01 - 9.47e-01 3.17e-01f 1 + 6 2.8914676e+04 1.56e+00 7.90e+01 -1.0 1.48e+01 - 3.97e-01 1.26e-01f 1 + 7 2.8108655e+04 6.74e-01 5.46e+01 -1.0 5.25e+00 - 9.65e-01 5.69e-01f 1 + 8 2.7648421e+04 1.63e-02 2.21e+01 -1.0 1.66e+00 - 8.62e-01 1.00e+00f 1 + 9 2.7654216e+04 1.36e-03 1.97e-01 -1.0 3.99e-01 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 10 2.7585786e+04 1.54e-04 9.33e-02 -1.7 3.31e-01 - 1.00e+00 1.00e+00f 1 - 11 2.7569878e+04 1.44e-03 5.82e-02 -2.5 6.72e-02 - 9.88e-01 1.00e+00f 1 - 12 2.7563905e+04 1.32e-02 2.35e-02 -2.5 1.04e-01 - 1.00e+00 1.00e+00h 1 - 13 2.7564551e+04 3.76e-05 1.44e-04 -2.5 2.12e-02 - 1.00e+00 1.00e+00h 1 - 14 2.7558777e+04 6.47e-03 6.79e-02 -3.8 7.82e-02 - 8.63e-01 1.00e+00f 1 - 15 2.7557880e+04 4.68e-03 1.05e-01 -3.8 7.97e-02 - 1.00e+00 3.39e-01h 1 - 16 2.7557754e+04 5.40e-04 4.26e-01 -3.8 1.03e-01 - 1.00e+00 8.88e-01h 1 - 17 2.7557736e+04 9.85e-06 2.13e-04 -3.8 2.55e-02 - 1.00e+00 1.00e+00h 1 - 18 2.7557602e+04 1.37e-05 1.54e-01 -5.7 3.45e-02 - 9.15e-01 7.81e-01h 1 - 19 2.7557572e+04 3.77e-06 1.09e-02 -5.7 9.25e-03 - 1.00e+00 9.65e-01h 1 + 10 2.7574726e+04 1.60e-04 2.22e+00 -2.5 2.75e-01 - 8.77e-01 1.00e+00f 1 + 11 2.7569519e+04 2.59e-03 1.09e-02 -2.5 7.33e-02 - 1.00e+00 1.00e+00f 1 + 12 2.7560217e+04 1.93e-02 1.30e-01 -3.8 1.26e-01 - 7.37e-01 1.00e+00h 1 + 13 2.7557763e+04 1.06e-02 2.11e-02 -3.8 1.04e-01 - 9.63e-01 6.25e-01h 1 + 14 2.7557837e+04 1.74e-03 6.51e-01 -3.8 9.82e-02 - 1.00e+00 8.38e-01h 1 + 15 2.7557760e+04 1.86e-05 4.99e-04 -3.8 2.51e-02 - 1.00e+00 1.00e+00h 1 + 16 2.7557607e+04 1.53e-05 1.57e-01 -5.7 3.40e-02 - 9.19e-01 7.80e-01h 1 + 17 2.7557573e+04 3.73e-06 1.16e-02 -5.7 9.12e-03 - 1.00e+00 9.63e-01h 1 + 18 2.7557573e+04 1.95e-06 9.06e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 + 19 2.7557573e+04 6.41e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 20 2.7557572e+04 1.94e-06 8.78e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 - 21 2.7557572e+04 6.42e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 - 22 2.7557571e+04 2.07e-07 1.75e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 - 23 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 + 20 2.7557571e+04 2.07e-07 1.73e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 + 21 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 -Number of Iterations....: 23 +Number of Iterations....: 21 (scaled) (unscaled) -Objective...............: 1.1868829499149242e+03 2.7557570559622669e+04 -Dual infeasibility......: 2.1624874764514693e-07 5.0209585722739846e-06 -Constraint violation....: 5.1921755382135226e-08 5.1921755382135226e-08 -Variable bound violation: 1.1735202365414921e-08 1.1735202365414921e-08 -Complementarity.........: 2.0827918551588903e-07 4.8359177721495380e-06 -Overall NLP error.......: 2.1624874764514693e-07 5.0209585722739846e-06 +Objective...............: 1.1868829484688854e+03 2.7557570526047901e+04 +Dual infeasibility......: 2.1611411147490891e-07 5.0178325304334832e-06 +Constraint violation....: 5.1889360219194158e-08 5.1889360219194158e-08 +Variable bound violation: 1.2335941157459729e-08 1.2335941157459729e-08 +Complementarity.........: 2.0818450105630093e-07 4.8337193467056586e-06 +Overall NLP error.......: 2.1611411147490891e-07 5.0178325304334832e-06 -Number of objective function evaluations = 25 -Number of objective gradient evaluations = 24 -Number of equality constraint evaluations = 25 -Number of inequality constraint evaluations = 25 -Number of equality constraint Jacobian evaluations = 24 -Number of inequality constraint Jacobian evaluations = 24 -Number of Lagrangian Hessian evaluations = 23 -Total seconds in IPOPT = 0.063 +Number of objective function evaluations = 23 +Number of objective gradient evaluations = 22 +Number of equality constraint evaluations = 23 +Number of inequality constraint evaluations = 23 +Number of equality constraint Jacobian evaluations = 22 +Number of inequality constraint Jacobian evaluations = 22 +Number of Lagrangian Hessian evaluations = 21 +Total seconds in IPOPT = 0.021 EXIT: Optimal Solution Found. ============================================================= @@ -551,7 +549,7 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 51 1 COAL 1.36 0.53 1.36 4.53 -0.55 2.11 52 1 COAL 1.36 0.65 1.36 4.53 -0.55 2.11 53 1 COAL 2.72 0.70 2.72 9.07 -1.11 4.23 -65 1 COAL 86.50 -4.97 86.50 86.50 -21.66 32.04 +65 1 WIND 86.50 -4.97 0.00 86.50 -21.66 32.04 67 1 COAL 1.41 -0.57 1.41 4.70 -0.57 2.19 68 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 69 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 @@ -559,39 +557,39 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 71 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 72 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 73 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 -76 1 COAL 1.20 0.52 1.20 4.00 -0.44 2.04 -77 1 COAL 0.72 0.26 0.72 2.40 -0.27 1.22 -78 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -79 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -90 1 COAL 0.96 -0.36 0.96 3.20 -0.36 1.63 -91 1 COAL 1.50 -0.55 1.50 5.00 -0.55 2.55 -92 0 COAL 0.00 0.00 1.89 6.30 -0.70 3.21 +76 1 NG 1.20 0.52 1.20 4.00 -0.44 2.04 +77 1 NG 0.72 0.26 0.72 2.40 -0.27 1.22 +78 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +79 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +90 1 NG 0.96 -0.36 0.96 3.20 -0.36 1.63 +91 1 NG 1.50 -0.55 1.50 5.00 -0.55 2.55 +92 0 NG 0.00 0.00 1.89 6.30 -0.70 3.21 94 1 COAL 5.40 -2.20 5.40 18.00 -2.20 8.39 -104 1 COAL 67.60 0.80 67.60 67.60 -14.26 21.09 -105 1 COAL 154.80 6.45 154.80 154.80 -28.51 42.17 -114 1 COAL 1.40 0.30 1.40 1.40 -0.24 0.36 -115 1 COAL 133.50 23.61 133.50 133.50 -21.60 31.95 +104 1 WIND 67.60 0.80 0.00 67.60 -14.26 21.09 +105 1 WIND 154.80 6.45 0.00 154.80 -28.51 42.17 +114 1 WIND 1.40 0.30 0.00 1.40 -0.24 0.36 +115 1 WIND 133.50 23.61 0.00 133.50 -21.60 31.95 125 1 COAL 39.02 6.39 39.02 130.05 -15.87 60.60 126 1 COAL 39.02 16.14 39.02 130.05 -15.87 60.60 127 1 COAL 39.02 22.98 39.02 130.05 -15.87 60.60 135 1 COAL 133.92 26.66 133.92 446.40 -54.46 208.02 136 1 COAL 133.92 25.72 133.92 446.40 -54.46 208.02 -147 1 COAL 92.40 11.12 92.40 92.40 -14.47 21.41 +147 1 WIND 92.40 11.12 0.00 92.40 -14.47 21.41 151 1 COAL 1.62 -0.61 1.62 5.40 -0.66 2.52 152 1 COAL 23.17 -5.34 23.17 77.22 -9.42 35.99 153 1 COAL 23.17 -1.87 23.17 77.22 -9.42 35.99 154 1 COAL 23.17 -5.54 23.17 77.22 -9.42 35.99 155 1 COAL 23.17 -5.55 23.17 77.22 -9.42 35.99 -161 0 COAL 0.00 0.00 41.58 138.60 -15.38 70.55 -164 0 COAL 0.00 0.00 3.60 12.00 -1.33 6.11 -165 0 COAL 0.00 0.00 7.80 26.00 -2.89 13.23 -166 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -167 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 -168 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -169 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -170 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 +161 0 NG 0.00 0.00 41.58 138.60 -15.38 70.55 +164 0 NG 0.00 0.00 3.60 12.00 -1.33 6.11 +165 0 NG 0.00 0.00 7.80 26.00 -2.89 13.23 +166 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +167 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 +168 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +169 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +170 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 182 1 COAL 5.25 0.88 5.25 17.50 -2.14 8.16 183 1 COAL 7.98 1.99 7.98 26.60 -3.25 12.40 -189 1 COAL 383.40 -46.67 170.75 569.15 -46.67 209.45 -196 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 -197 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 +189 1 NUCLEAR 383.40 -46.67 170.75 569.15 -46.67 209.45 +196 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 +197 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 diff --git a/agentigrid/tests/results/test_opflow_parser.py b/agentigrid/tests/results/test_opflow_parser.py index f0cedf0c..54a9b42f 100644 --- a/agentigrid/tests/results/test_opflow_parser.py +++ b/agentigrid/tests/results/test_opflow_parser.py @@ -51,10 +51,10 @@ def test_objective_type(self, result: OPFLOWResult): assert result.objective_type == "MIN_GEN_COST" def test_num_iterations(self, result: OPFLOWResult): - assert result.num_iterations == 23 + assert result.num_iterations == 21 def test_solve_time(self, result: OPFLOWResult): - assert result.solve_time == pytest.approx(0.042, abs=0.001) + assert result.solve_time == pytest.approx(0.021, abs=0.001) def test_bus_count(self, result: OPFLOWResult): assert len(result.buses) == 200 @@ -91,10 +91,18 @@ def test_branch_187_189_highest_sf(self, result: OPFLOWResult): assert abs(max_br.Sf - 387.19) < 0.01 def test_gen_189(self, result: OPFLOWResult): - """Gen at bus 189: Pg≈383.40, fuel=COAL.""" + """Gen at bus 189: Pg≈383.40, fuel=NUCLEAR (mpc.genfuel of case_ACTIVSg200.m).""" gen189 = next(g for g in result.generators if g.bus == 189) assert abs(gen189.Pg - 383.40) < 0.01 - assert gen189.fuel == "COAL" + assert gen189.fuel == "NUCLEAR" + + def test_fuel_mix_matches_case_file(self, result: OPFLOWResult): + """All 49 units keep the fuel from mpc.genfuel (25 coal, 17 ng, 6 wind, + 1 nuclear). A fixture regenerated with a writer that drops the "};" + after mpc.genfuel would show every unit as COAL.""" + from collections import Counter + assert Counter(g.fuel for g in result.generators) == { + "COAL": 25, "NG": 17, "WIND": 6, "NUCLEAR": 1} # =========================================================================== diff --git a/agentigrid/tests/runner/test_executor.py b/agentigrid/tests/runner/test_executor.py index 1b46cbd5..3c89c92b 100644 --- a/agentigrid/tests/runner/test_executor.py +++ b/agentigrid/tests/runner/test_executor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import shutil from pathlib import Path from unittest.mock import patch, MagicMock @@ -331,6 +332,8 @@ def test_no_mpirun_for_sopflow_with_mpi_np_1(self, tmp_path: Path): assert cmd_arg[0] != "mpirun" +@pytest.mark.skipif(not _has_opflow, reason="opflow binary not available") +@pytest.mark.skipif(not _has_test_file, reason="case_ACTIVSg200.m not in data/exago/examples/") class TestExecutorLive: def test_opflow_run(self, tmp_path: Path): @@ -350,8 +353,11 @@ def test_opflow_run(self, tmp_path: Path): assert result.elapsed_seconds < exago.timeout assert result.exit_code == 0 or result.success # MPI noise tolerance - # Save stdout for Step 1.6 (only if converged, to avoid overwriting good sample) - if "Optimal Solution Found" in result.stdout: + # Regenerate the parser fixture only on request: the parser tests in + # tests/results/ hard-code values from it, so overwriting it on every + # live run made their results depend on the machine and test order. + # AGENTIGRID_REGENERATE_FIXTURES=1 python -m pytest tests/runner/test_executor.py -k test_opflow_run + if os.environ.get("AGENTIGRID_REGENERATE_FIXTURES") == "1" and "Optimal Solution Found" in result.stdout: sample_path = Path(__file__).resolve().parent.parent / "fixtures" / "sample_opflow_output.txt" sample_path.write_text(result.stdout, encoding="utf-8") print(f"\nSaved sample output to {sample_path} ({len(result.stdout)} bytes)") From 2677d59bc49e4c9e17d72e8bbe3f51ed4ea642a1 Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Tue, 6 Oct 2026 20:46:20 +0200 Subject: [PATCH 3/5] AgentiGrid MATPOWER reader/writer: reject malformed input, stable round trips Five defects found while specifying the parser and writer: 1. Rows with a non-numeric value were skipped with only a log warning, so a bus, generator or branch could silently disappear; too-short rows crashed with IndexError. Known sections (bus, gen, branch, gencost) now raise ValueError naming the section and row for non-numeric values, column counts outside the section's limits, and non-integral values in integer columns. 2. Valid MATLAB syntax was misread: two rows on one line became one garbage row and comma-separated rows were all dropped. These, '...' continuations, unterminated blocks and a missing mpc.bus now raise ValueError. 3. The writer crashed on Inf/NaN, which the reader accepts. They are written as Inf/-Inf/NaN; bus solution columns no longer lose precision to .4f. 4. Every read/write round trip added another "MATPOWER Case Format" header line. Writing is now idempotent. 5. Case version '1' was accepted although ExaGO reads version 2; it now raises. A missing version still means '2'. CRLF input is normalised on read and output is always LF. The function line always carries network.casename. All 29 case files in datafiles/ still parse and round-trip to equivalent networks. New tests/grid_files/test_matpower_hardening.py (29 tests, 22 fail without the fix). Co-Authored-By: Claude Opus 5.5 --- .../agentigrid/parsers/matpower_parser.py | 132 +++++++--- .../agentigrid/parsers/matpower_writer.py | 39 ++- agentigrid/claude_code_changes.md | 22 ++ .../grid_files/test_matpower_hardening.py | 230 ++++++++++++++++++ 4 files changed, 382 insertions(+), 41 deletions(-) create mode 100644 agentigrid/tests/grid_files/test_matpower_hardening.py diff --git a/agentigrid/agentigrid/parsers/matpower_parser.py b/agentigrid/agentigrid/parsers/matpower_parser.py index ff50a626..94c0f498 100644 --- a/agentigrid/agentigrid/parsers/matpower_parser.py +++ b/agentigrid/agentigrid/parsers/matpower_parser.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import math import re from pathlib import Path @@ -26,6 +27,25 @@ # Match mpc.
= [ or mpc.
= { _SECTION_RE = re.compile(r"^mpc\.(\w+)\s*=\s*([{\[])", re.MULTILINE) +# Only MATPOWER case format version 2 is supported (ExaGO reads version 2). +_SUPPORTED_VERSIONS = ("2",) + +# Sections parsed into dataclasses; any other section is kept as raw text. +_KNOWN_SECTIONS = ("bus", "gen", "branch", "gencost") +# Column limits per known section. Bus rows have 13 input columns plus up to +# 4 optional solution columns (lam_P, lam_Q, mu_Vmax, mu_Vmin); gen and +# branch rows may carry any number of extra columns, which are kept. +_MIN_COLS = {"bus": 13, "gen": 10, "branch": 13, "gencost": 4} +_MAX_COLS = {"bus": 17} +# 0-based columns that must hold integers (MATPOWER BUS_I, BUS_TYPE, +# BUS_AREA, ZONE; GEN_BUS, GEN_STATUS; F_BUS, T_BUS, BR_STATUS; MODEL, NCOST). +_INT_COLS = { + "bus": (0, 1, 6, 10), + "gen": (0, 7), + "branch": (0, 1, 10), + "gencost": (0, 3), +} + def _parse_float(s: str) -> float: """Parse a string as float, handling MATLAB Inf/NaN.""" @@ -39,21 +59,61 @@ def _parse_float(s: str) -> float: return float(sl) -def _split_data_row(line: str) -> list[str]: - """Split a MATPOWER data row into tokens, stripping comments and semicolons.""" - # Remove inline comment - idx = line.find("%") - if idx >= 0: - line = line[:idx] - # Remove trailing semicolons and whitespace - line = line.rstrip().rstrip(";").rstrip() - return line.split() - +def _parse_rows(section: str, block: str) -> list[list[float]]: + """Parse the rows of a known numeric section (one row per line). -def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, int]: + Anything that would otherwise be misread raises ``ValueError`` naming the + section and the 1-based data row: unsupported syntax (several rows on one + line, commas, ``...`` continuations), non-numeric values, a column count + outside the section's limits, or a non-integral value in an integer column. + Rows are never skipped silently. + """ + rows: list[list[float]] = [] + for line in block.split("\n"): + idx = line.find("%") + if idx >= 0: + line = line[:idx] + line = line.strip().rstrip(";").rstrip() + if not line: + continue + where = f"mpc.{section} row {len(rows) + 1}" + if "..." in line: + raise ValueError(f"{where}: line continuation '...' is not supported") + if ";" in line: + raise ValueError( + f"{where}: several rows on one line are not supported; " + "put each row on its own line" + ) + if "," in line: + raise ValueError( + f"{where}: comma-separated values are not supported; " + "separate values with spaces or tabs" + ) + try: + values = [_parse_float(t) for t in line.split()] + except ValueError: + raise ValueError(f"{where}: non-numeric value in {line!r}") from None + lo, hi = _MIN_COLS[section], _MAX_COLS.get(section) + if len(values) < lo or (hi is not None and len(values) > hi): + expected = f"{lo}-{hi}" if hi is not None else f"at least {lo}" + raise ValueError(f"{where}: {len(values)} columns, expected {expected}") + for col in _INT_COLS[section]: + v = values[col] + if not math.isfinite(v) or v != int(v): + raise ValueError(f"{where}: column {col + 1} must be an integer, got {v!r}") + rows.append(values) + return rows + + +def _extract_section_block( + text: str, start: int, bracket: str, section: str = "?" +) -> tuple[str, int]: """Extract text from *start* until the matching closing bracket. Returns (block_content, end_position). + + Raises: + ValueError: If the block is never closed. """ close = "]" if bracket == "[" else "}" # Find the end of the section @@ -66,6 +126,8 @@ def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, in elif ch == close: depth -= 1 i += 1 + if depth > 0: + raise ValueError(f"mpc.{section}: '{bracket}' block is not terminated") return text[start:i - 1], i @@ -80,6 +142,11 @@ def _collect_section_comment(text: str, section_start: int) -> str: else: break comment_lines.reverse() + # Drop surrounding blank lines so a read/write round trip is stable. + while comment_lines and not comment_lines[0].strip(): + comment_lines.pop(0) + while comment_lines and not comment_lines[-1].strip(): + comment_lines.pop() return "\n".join(comment_lines) @@ -180,7 +247,8 @@ def parse_matpower(path: Path) -> MATNetwork: if not path.exists(): raise FileNotFoundError(f"MATPOWER file not found: {path}") - text = path.read_text(encoding="utf-8") + # Normalise line endings: CRLF (Windows-authored) files read the same as LF. + text = path.read_text(encoding="utf-8").replace("\r\n", "\n") # --- Function name --- m = _FUNC_RE.search(text) @@ -191,6 +259,11 @@ def parse_matpower(path: Path) -> MATNetwork: # --- Version --- m = _VERSION_RE.search(text) version = m.group(1) if m else "2" + if version not in _SUPPORTED_VERSIONS: + raise ValueError( + f"Unsupported MATPOWER case version '{version}' in {path}; " + f"supported: {', '.join(_SUPPORTED_VERSIONS)}" + ) # --- baseMVA --- m = _BASEMVA_RE.search(text) @@ -222,12 +295,13 @@ def parse_matpower(path: Path) -> MATNetwork: if section_name in ("version", "baseMVA"): continue - block_content, _ = _extract_section_block(text, block_start, bracket) + block_content, _ = _extract_section_block(text, block_start, bracket, section_name) # Collect preceding comment line_start = text.rfind("\n", 0, match.start()) + 1 comment = _collect_section_comment(text, line_start) _section_comments[section_name] = comment + prefix = comment + "\n" if comment else "" if bracket == "{": # Cell array (bus_name, gentype, genfuel, etc.) → store as raw @@ -235,23 +309,19 @@ def parse_matpower(path: Path) -> MATNetwork: # mpc.genfuel by searching for "};", so a bare "}" makes it skip the # fuel types (and with them the default ramp rates and the # renewable Pmin=0 relaxation). - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "};" + raw_text = prefix + text[match.start():block_start] + block_content + "};" + extra_sections[section_name] = raw_text + continue + + if section_name not in _KNOWN_SECTIONS: + # Unknown numeric section (areas, dcline, ...) → store as raw + raw_text = prefix + text[match.start():block_start] + block_content + "];" extra_sections[section_name] = raw_text + logger.debug("Stored unknown section mpc.%s as raw text", section_name) continue - # Numeric matrix section — parse rows - rows: list[list[float]] = [] - for line in block_content.split("\n"): - stripped = line.strip() - if not stripped or stripped.startswith("%"): - continue - tokens = _split_data_row(line) - if not tokens: - continue - try: - rows.append([_parse_float(t) for t in tokens]) - except ValueError: - logger.warning("Skipping unparseable row in mpc.%s: %s", section_name, line.strip()) + # Known numeric matrix section — parse rows (raises on bad input) + rows = _parse_rows(section_name, block_content) if section_name == "bus": buses = [_row_to_bus(r) for r in rows] @@ -265,11 +335,9 @@ def parse_matpower(path: Path) -> MATNetwork: elif section_name == "gencost": gencost = [_row_to_gencost(r) for r in rows] logger.debug("Parsed %d gencost entries", len(gencost)) - else: - # Unknown numeric section → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "];" - extra_sections[section_name] = raw_text - logger.debug("Stored unknown section mpc.%s as raw text", section_name) + + if "bus" not in _section_comments: + raise ValueError(f"Cannot find mpc.bus in {path}") # Store section comments on the network for the writer net = MATNetwork( diff --git a/agentigrid/agentigrid/parsers/matpower_writer.py b/agentigrid/agentigrid/parsers/matpower_writer.py index 181ce579..e27a08c8 100644 --- a/agentigrid/agentigrid/parsers/matpower_writer.py +++ b/agentigrid/agentigrid/parsers/matpower_writer.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +import math +import re from pathlib import Path from agentigrid.parsers.matpower_model import ( @@ -15,9 +17,19 @@ logger = logging.getLogger("agentigrid.parsers.matpower") +_FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*\w+", re.MULTILINE) + def _fmt(value: float) -> str: - """Format a numeric value, writing integers without decimals.""" + """Format a numeric value, writing integers without decimals. + + Infinities and NaN are written as MATLAB ``Inf`` / ``-Inf`` / ``NaN`` + (the parser reads them back). + """ + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Inf" if value > 0 else "-Inf" if value == int(value) and abs(value) < 1e15: return str(int(value)) return f"{value:.10g}" @@ -31,7 +43,7 @@ def _write_bus_row(b: Bus) -> str: ] for val in (b.lam_P, b.lam_Q, b.mu_Vmax, b.mu_Vmin): if val is not None: - parts.append(f"{val:.4f}") + parts.append(_fmt(val)) return "\t" + "\t".join(parts) + ";" @@ -93,14 +105,22 @@ def write_matpower(network: MATNetwork, path: Path) -> None: lines: list[str] = [] # --- Header --- - lines.append(network.header_comments.rstrip("\n")) - # Ensure function line is present (header_comments includes it) - if f"function mpc = {network.casename}" not in network.header_comments: - lines.append(f"function mpc = {network.casename}") + # The function line always carries network.casename: an existing one in + # header_comments is renamed, a missing one is added as the first line. + header = network.header_comments.rstrip("\n") + func_line = f"function mpc = {network.casename}" + if _FUNC_RE.search(header): + header = _FUNC_RE.sub(lambda _m: func_line, header, count=1) + else: + header = func_line + ("\n" + header if header else "") + lines.append(header) # --- Version & baseMVA --- - lines.append("") - lines.append(f"%% MATPOWER Case Format : Version {network.version}") + # The header read back from a file already holds the format comment; + # adding it again would grow the file on every read/write round trip. + if "MATPOWER Case Format" not in header: + lines.append("") + lines.append(f"%% MATPOWER Case Format : Version {network.version}") lines.append(f"mpc.version = '{network.version}';") lines.append("") lines.append("%%----- Power Flow Data -----%%") @@ -154,5 +174,6 @@ def write_matpower(network: MATNetwork, path: Path) -> None: lines.append("") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") + # Always LF line endings, whatever the platform or the source file used. + path.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") logger.info("Wrote MATPOWER file: %s", path) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 7a4bece1..97e98b79 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,28 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## MATPOWER reader/writer hardening — no silent misreads, stable round trips (2026-10-05) + +Found while writing an independent spec for the parser and writer (round-trip, error and syntax rules). Every real case in ExaGO's `datafiles/` (29 case files, `case5` to `case_ACTIVSg10k`, including CRLF ones) still parses and round-trips to an equivalent network; the files are already accepted by the stricter rules. + +| # | Defect | Fix | +|---|---|---| +| 1 | A row with a non-numeric value was skipped with only a log warning, so a bus, generator or branch could silently vanish. Too-short rows crashed with `IndexError`. | `_parse_rows()` raises `ValueError` naming the section and 1-based row for non-numeric values, column counts outside the section's limits (bus 13–17; gen ≥ 10; branch ≥ 13; gencost ≥ 4) and non-integral/Inf/NaN values in integer columns. | +| 2 | Valid MATLAB syntax was misread: two rows on one line became one garbage row; comma-separated rows were all dropped; `...` continuations crashed. | These raise `ValueError` with a clear message. An unterminated `[`/`{` block and a missing `mpc.bus` raise as well. | +| 3 | The writer crashed on `Inf`/`NaN` (`OverflowError`/`ValueError`), which the reader accepts. | Written as `Inf`, `-Inf`, `NaN`. Bus solution columns (`lam_*`, `mu_*`) use the same 10-significant-digit format instead of `.4f`. | +| 4 | Every read/write round trip added another `%% MATPOWER Case Format` line, and blank lines accumulated before raw sections. | The format comment is written only if the header lacks it; collected section comments drop surrounding blank lines. Writing is idempotent. | +| 5 | MATPOWER case version `'1'` was accepted although ExaGO reads version 2. | Raises `ValueError`; a missing version still means `'2'`. | + +Also: CRLF input is normalised on read and the writer always emits LF; the `function mpc = …` line always carries `network.casename` (renamed if the header has another name, added first if missing). Unknown numeric sections (`areas`, `dcline`, …) stay opaque raw text and are not validated. + +| File | Change | +|---|---| +| `agentigrid/parsers/matpower_parser.py` | Fixes 1, 2, 4 (comments), 5; CRLF normalisation | +| `agentigrid/parsers/matpower_writer.py` | Fixes 3, 4 (header); casename/function line; LF output | +| `tests/grid_files/test_matpower_hardening.py` | New (29 tests; 22 fail without the fix) | + +--- + ## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) **Symptom.** SCOPFLOW reported local infeasibility on AgentiGrid-rewritten ACTIVSg200 at base load, while the original file converged. The rewritten file had `mpc.genfuel = { ... }` with a bare `}`. diff --git a/agentigrid/tests/grid_files/test_matpower_hardening.py b/agentigrid/tests/grid_files/test_matpower_hardening.py new file mode 100644 index 00000000..fcad55e4 --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_hardening.py @@ -0,0 +1,230 @@ +"""MATPOWER reader/writer hardening: no silent misreads, stable round trips. + +Covers five defects found while specifying the parser: + 1. malformed rows were skipped with only a log warning (a bus could vanish); + 2. valid-but-unsupported MATLAB syntax was misread (two rows on one line + became one garbage row; comma-separated rows were all dropped); + 3. the writer crashed on Inf/NaN, which the reader accepts; + 4. every read/write round trip added another header line; + 5. MATPOWER case version '1' was accepted although ExaGO reads version 2. +""" + +from __future__ import annotations + +import copy +import math +from pathlib import Path + +import pytest + +from agentigrid.parsers.matpower_model import Branch, Bus, GenCost, Generator, MATNetwork +from agentigrid.parsers.matpower_parser import parse_matpower +from agentigrid.parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + +BUS = "\t1\t3\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;\n\t2\t1\t90\t30\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" +GEN = "\t1\t0\t0\t300\t-300\t1\t100\t1\t250\t10;" +BRANCH = "\t1\t2\t0.01\t0.1\t0\t100\t100\t100\t0\t0\t1\t-360\t360;" +GENCOST = "\t2\t0\t0\t3\t0.11\t5\t150;" + + +def _case(*, version="2", bus=BUS, gen=GEN, branch=BRANCH, gencost=GENCOST, extra="", + drop: tuple[str, ...] = ()) -> str: + parts = { + "head": "function mpc = tiny\n%TINY test case\n\n%% MATPOWER Case Format : Version 2\n", + "version": f"mpc.version = '{version}';\n", + "base": "\n%% system MVA base\nmpc.baseMVA = 100;\n", + "bus": f"\n%% bus data\nmpc.bus = [\n{bus}\n];\n", + "gen": f"\n%% generator data\nmpc.gen = [\n{gen}\n];\n", + "branch": f"\n%% branch data\nmpc.branch = [\n{branch}\n];\n", + "gencost": f"\n%% generator cost data\nmpc.gencost = [\n{gencost}\n];\n", + "extra": f"\n{extra}\n" if extra else "", + } + return "".join(v for k, v in parts.items() if k not in drop) + + +def _write(tmp_path: Path, text: str, name: str = "c.m", crlf: bool = False) -> Path: + p = tmp_path / name + data = text.replace("\n", "\r\n") if crlf else text + p.write_bytes(data.encode("utf-8")) + return p + + +# --- 5. version ------------------------------------------------------------ + +def test_version_1_rejected(tmp_path): + with pytest.raises(ValueError, match="version"): + parse_matpower(_write(tmp_path, _case(version="1"))) + + +def test_missing_version_defaults_to_2(tmp_path): + text = _case().replace("mpc.version = '2';\n", "") + assert parse_matpower(_write(tmp_path, text)).version == "2" + + +# --- 1./2. malformed rows and unsupported syntax --------------------------- + +@pytest.mark.parametrize( + "kwargs, section", + [ + ({"bus": BUS.replace("\n", " ")}, "bus"), # two rows, one line + ({"bus": BUS.replace("\t", ",")}, "bus"), # commas + ({"gen": "\t1\t0\t0\t300\t-300 ...\n\t1\t100\t1\t250\t10;"}, "gen"), # continuation + ({"bus": BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;"}, "bus"), # non-numeric + ({"bus": "\t1\t3\t0\t0;"}, "bus"), # too few columns + ({"gen": "\t1\t0\t0;"}, "gen"), + ({"branch": "\t1\t2\t0.01;"}, "branch"), + ({"gencost": "\t2\t0\t0;"}, "gencost"), + ({"bus": BUS + "\n\t3" + "\t1" * 17 + ";"}, "bus"), # 18 bus columns + ({"bus": BUS.replace("\t1\t3\t", "\t1.5\t3\t", 1)}, "bus"), # non-integral bus_i + ({"gen": GEN.replace("\t100\t1\t", "\t100\tInf\t")}, "gen"), # Inf status + ({"branch": BRANCH.replace("\t0\t1\t-360", "\t0\tNaN\t-360")}, "branch"), # NaN status + ], + ids=["rows-one-line", "commas", "continuation", "non-numeric", "short-bus", "short-gen", + "short-branch", "short-gencost", "bus-18-cols", "int-col-1.5", "int-col-inf", + "int-col-nan"], +) +def test_malformed_input_raises_naming_section(tmp_path, kwargs, section): + with pytest.raises(ValueError, match=rf"mpc\.{section}\b"): + parse_matpower(_write(tmp_path, _case(**kwargs))) + + +def test_non_numeric_row_reports_row_number(tmp_path): + bad = BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" + with pytest.raises(ValueError, match=r"mpc\.bus row 3"): + parse_matpower(_write(tmp_path, _case(bus=bad))) + + +def test_unterminated_block_raises(tmp_path): + text = _case(drop=("gencost",)).replace("\n];\n\n%% branch data", "\n\n%% branch data") + with pytest.raises(ValueError, match="terminated"): + parse_matpower(_write(tmp_path, text)) + + +def test_missing_bus_section_raises(tmp_path): + with pytest.raises(ValueError, match="mpc.bus"): + parse_matpower(_write(tmp_path, _case(drop=("bus",)))) + + +def test_missing_gen_and_branch_give_empty_lists(tmp_path): + net = parse_matpower(_write(tmp_path, _case(drop=("gen", "branch", "gencost")))) + assert net.generators == [] and net.branches == [] and net.gencost == [] + + +def test_partial_bus_solution_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5;", 1)))) + b = net.buses[0] + assert b.lam_P == 6.5 and b.lam_Q is None and b.mu_Vmax is None and b.mu_Vmin is None + + +def test_integral_floats_accepted_in_integer_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("\t1\t3\t", "\t1.0\t3.0\t", 1)))) + assert net.buses[0].bus_i == 1 and isinstance(net.buses[0].bus_i, int) + + +# --- line endings ------------------------------------------------------------ + +def test_crlf_reads_like_lf_and_writer_emits_lf(tmp_path): + lf = parse_matpower(_write(tmp_path, _case(), "lf.m")) + crlf = parse_matpower(_write(tmp_path, _case(), "crlf.m", crlf=True)) + assert crlf.buses == lf.buses and crlf.header_comments == lf.header_comments + out = tmp_path / "out.m" + write_matpower(crlf, out) + assert b"\r" not in out.read_bytes() + + +# --- 3. Inf / NaN ---------------------------------------------------------- + +def test_inf_nan_round_trip(tmp_path): + net = parse_matpower(_write(tmp_path, _case())) + net.branches[0].rateA = math.inf + net.generators[0].Qmin = -math.inf + net.buses[1].Pd = math.nan + out = tmp_path / "out.m" + write_matpower(net, out) + back = parse_matpower(out) + assert back.branches[0].rateA == math.inf + assert back.generators[0].Qmin == -math.inf + assert math.isnan(back.buses[1].Pd) + + +def test_solution_columns_keep_precision(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5\t0\t1e-07\t0.000123456789;", 1)))) + out = tmp_path / "out.m" + write_matpower(net, out) + b = parse_matpower(out).buses[0] + assert math.isclose(b.mu_Vmax, 1e-07, rel_tol=1e-9) + assert math.isclose(b.mu_Vmin, 0.000123456789, rel_tol=1e-9) + + +# --- 4. idempotent writing --------------------------------------------------- + +EXTRAS = ( + "%% generator fuel type\nmpc.genfuel = {\n\t'coal';\n};\n\n" + "mpc.areas = [\n\t1\t1;\n];" +) + + +def test_round_trip_text_is_idempotent(tmp_path): + src = _write(tmp_path, _case(extra=EXTRAS), crlf=True) + a, b, c = (tmp_path / n for n in ("a.m", "b.m", "c.m")) + write_matpower(parse_matpower(src), a) + write_matpower(parse_matpower(a), b) + write_matpower(parse_matpower(b), c) + assert a.read_text() == b.read_text() == c.read_text() + assert a.read_text().count("MATPOWER Case Format") == 1 + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_real_case_round_trip_is_idempotent(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert a.read_text() == b.read_text() + + +def test_section_comments_stay_with_their_section(tmp_path): + out = tmp_path / "out.m" + write_matpower(parse_matpower(_write(tmp_path, _case(extra=EXTRAS))), out) + text = out.read_text() + assert text.index("%% generator data") < text.index("mpc.gen =") + assert text.index("];", text.index("mpc.bus =")) < text.index("%% generator data") + assert text.index("%% generator fuel type") < text.index("mpc.genfuel =") + + +# --- casename / function line ------------------------------------------------ + +def _net(header: str, casename: str = "mine") -> MATNetwork: + return MATNetwork( + casename=casename, version="2", baseMVA=100.0, + buses=[Bus(1, 3, 0.0, 0.0, 0.0, 0.0, 1, 1.0, 0.0, 230.0, 1, 1.1, 0.9)], + generators=[Generator(1, 0.0, 0.0, 300.0, -300.0, 1.0, 100.0, 1, 250.0, 10.0)], + branches=[Branch(1, 1, 0.01, 0.1, 0.0, 100.0, 100.0, 100.0, 0.0, 0.0, 1, -360.0, 360.0)], + gencost=[GenCost(2, 0.0, 0.0, 3, [0.11, 5.0, 150.0])], + header_comments=header, + ) + + +def test_casename_wins_over_header_function_line(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net("function mpc = other\n%comment\n", casename="mine"), out) + text = out.read_text() + assert "function mpc = mine" in text and "function mpc = other" not in text + assert text.count("function mpc") == 1 + assert parse_matpower(out).casename == "mine" + + +def test_empty_header_writes_function_line_first(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net(""), out) + assert out.read_text().splitlines()[0] == "function mpc = mine" + assert parse_matpower(out).casename == "mine" + + +def test_writer_does_not_mutate_its_input(tmp_path): + net = parse_matpower(_write(tmp_path, _case(extra=EXTRAS))) + before = copy.deepcopy(net) + write_matpower(net, tmp_path / "out.m") + assert net == before From a8319021d5162df4a85a509984a6bca44745588a Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Tue, 6 Oct 2026 20:46:20 +0200 Subject: [PATCH 4/5] AgentiGrid MATPOWER writer/reader: exact near-max floats, idempotent rounding, baseMVA literals Three defects found by property-based round-trip testing: 1. A finite value near the float maximum was written with 10 significant digits that round past it and read back as Inf. The writer now falls back to the exact representation when rounding overflows. 2. A value that rounds to an integer (e.g. 9999999999.5) was written as "1e+10" the first time and "10000000000" after re-reading, so writing was not idempotent. The integer form is now decided on the rounded value. 3. mpc.baseMVA was read with a digits-and-dots pattern: "1e2" (and "1e+15", which the writer itself emits for large values) was silently read as 1, and a negative value was reported as missing. Any numeric literal (sign, exponent, Inf/NaN) on an mpc.baseMVA line is now accepted, text in comments is ignored, and a non-numeric value raises ValueError. 18 more tests in tests/grid_files/test_matpower_hardening.py (12 fail without the fix). All 29 case files in datafiles/ still round-trip. Co-Authored-By: Claude Opus 5.5 --- .../agentigrid/parsers/matpower_parser.py | 9 +++- .../agentigrid/parsers/matpower_writer.py | 14 ++++-- agentigrid/claude_code_changes.md | 8 +++ .../grid_files/test_matpower_hardening.py | 49 +++++++++++++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/agentigrid/agentigrid/parsers/matpower_parser.py b/agentigrid/agentigrid/parsers/matpower_parser.py index 94c0f498..23b470e5 100644 --- a/agentigrid/agentigrid/parsers/matpower_parser.py +++ b/agentigrid/agentigrid/parsers/matpower_parser.py @@ -23,7 +23,8 @@ _FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*(\w+)", re.MULTILINE) _VERSION_RE = re.compile(r"mpc\.version\s*=\s*'([^']+)'") -_BASEMVA_RE = re.compile(r"mpc\.baseMVA\s*=\s*([\d.]+)") +# The value is any numeric token (sign, exponent, Inf/NaN) up to ";" or a comment. +_BASEMVA_RE = re.compile(r"^[ \t]*mpc\.baseMVA[ \t]*=[ \t]*([^;%\n]*)", re.MULTILINE) # Match mpc.
= [ or mpc.
= { _SECTION_RE = re.compile(r"^mpc\.(\w+)\s*=\s*([{\[])", re.MULTILINE) @@ -269,7 +270,11 @@ def parse_matpower(path: Path) -> MATNetwork: m = _BASEMVA_RE.search(text) if not m: raise ValueError(f"Cannot find mpc.baseMVA in {path}") - baseMVA = float(m.group(1)) + token = m.group(1).strip() + try: + baseMVA = _parse_float(token) + except ValueError: + raise ValueError(f"mpc.baseMVA: invalid value {token!r} in {path}") from None # --- Header comments (everything before mpc.version or mpc.baseMVA) --- first_mpc = text.find("mpc.") diff --git a/agentigrid/agentigrid/parsers/matpower_writer.py b/agentigrid/agentigrid/parsers/matpower_writer.py index e27a08c8..c402ea0a 100644 --- a/agentigrid/agentigrid/parsers/matpower_writer.py +++ b/agentigrid/agentigrid/parsers/matpower_writer.py @@ -30,9 +30,17 @@ def _fmt(value: float) -> str: return "NaN" if math.isinf(value): return "Inf" if value > 0 else "-Inf" - if value == int(value) and abs(value) < 1e15: - return str(int(value)) - return f"{value:.10g}" + text = f"{value:.10g}" + rounded = float(text) + if math.isinf(rounded): + # Rounding to 10 digits pushed a value near the float maximum past + # it; that text would read back as Inf. Write it exactly instead. + text, rounded = repr(value), value + # Decide the integer form on the ROUNDED value, so a value that rounds to + # an integer is written the same way the second time (idempotence). + if rounded == int(rounded) and abs(rounded) < 1e15: + return str(int(rounded)) + return text def _write_bus_row(b: Bus) -> str: diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 97e98b79..29513d3d 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -24,6 +24,14 @@ Also: CRLF input is normalised on read and the writer always emits LF; the `func | `agentigrid/parsers/matpower_writer.py` | Fixes 3, 4 (header); casename/function line; LF output | | `tests/grid_files/test_matpower_hardening.py` | New (29 tests; 22 fail without the fix) | +**Follow-up (property-based testing).** Randomised round-trip tests then found three more defects: + +- A finite value near the float maximum was written with 10 significant digits that round past it, and read back as `Inf`. The writer now falls back to the exact representation. +- A value that rounds to an integer (for example `9999999999.5`) was written as `1e+10` the first time and `10000000000` the second, so writing was not idempotent. The integer form is now decided on the rounded value. +- `mpc.baseMVA` was read with a digits-and-dots pattern. `1e2` (or `1e+15`, which the writer itself emits) was silently read as `1`, and negative values were "not found". It now accepts any numeric literal (sign, exponent, `Inf`/`NaN`) on an `mpc.baseMVA` line, ignores the text in comments, and raises `ValueError` for a non-numeric value. + +18 more tests; 12 of them fail without these fixes. + --- ## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) diff --git a/agentigrid/tests/grid_files/test_matpower_hardening.py b/agentigrid/tests/grid_files/test_matpower_hardening.py index fcad55e4..88f71b3c 100644 --- a/agentigrid/tests/grid_files/test_matpower_hardening.py +++ b/agentigrid/tests/grid_files/test_matpower_hardening.py @@ -228,3 +228,52 @@ def test_writer_does_not_mutate_its_input(tmp_path): before = copy.deepcopy(net) write_matpower(net, tmp_path / "out.m") assert net == before + + +# --- number formatting and baseMVA (found by property-based testing) --------- + +@pytest.mark.parametrize( + "value", + [9999999999.5, 123456789012.345, 1e15, 1e16, 0.1 + 0.2, 5e-324, + 1.7976931345e308, -1.7976931348623157e308], + ids=["rounds-to-int", "11-digits", "1e15", "1e16", "0.3", "subnormal", + "near-max", "-max"], +) +def test_number_round_trip_is_close_finite_and_idempotent(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.buses[1].Pd = value + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(net, a) + back = parse_matpower(a) + assert math.isfinite(back.buses[1].Pd) + assert math.isclose(back.buses[1].Pd, value, rel_tol=1e-9, abs_tol=1e-12) + write_matpower(back, b) + assert a.read_text() == b.read_text() + + +@pytest.mark.parametrize( + "text, expected", + [("1e2", 100.0), ("100.0", 100.0), ("-1", -1.0), ("1e+16", 1e16), ("Inf", math.inf)], +) +def test_basemva_accepts_any_numeric_literal(tmp_path, text, expected): + net = parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", f"mpc.baseMVA = {text};"))) + assert net.baseMVA == expected + + +def test_basemva_in_a_comment_is_ignored(tmp_path): + text = _case().replace("%TINY test case\n", "%TINY test case\n% mpc.baseMVA = 5;\n") + assert parse_matpower(_write(tmp_path, text)).baseMVA == 100.0 + + +def test_basemva_non_numeric_raises(tmp_path): + with pytest.raises(ValueError, match="baseMVA"): + parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", "mpc.baseMVA = abc;"))) + + +@pytest.mark.parametrize("value", [1e16, -1.0, 1.7976931345e308]) +def test_basemva_round_trip(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.baseMVA = value + out = tmp_path / "out.m" + write_matpower(net, out) + assert math.isclose(parse_matpower(out).baseMVA, value, rel_tol=1e-9) From 8732086485969302c87cf26c73c5c07c063540cf Mon Sep 17 00:00:00 2001 From: Amir Hajdar Date: Sat, 3 Oct 2026 19:36:47 +0000 Subject: [PATCH 5/5] AgentiGrid: robust agent iterations, accurate UI timeline, LLM usage accounting Agent loop - Each iteration runs in a guard: an exception in an action handler is discarded with feedback to the model and the run continues; sweep, boundary and reserve handlers tolerate a non-dict feasibility / candidate_set. - The UI callback fires only when the iteration recorded a journal entry; otherwise a UI-only "discarded" card says why. The final re-emit of the last entry is removed (it duplicated the last card and, after rejected iterations, showed a stale lower-numbered card after the last one). - journal.discarded_actions records iterations that produced no entry (kind rejected / api_error / truncated / internal, feedback, stop_reason, raw response capped at 4000 characters). Entries and statistics are unchanged. - Responses cut at max_tokens get their own message and feedback; a modify that applies no command says so; modify entries keep skipped_commands. - Optional stall breaker: AGENTIGRID_STALL_LIMIT=N stops after N consecutive iterations without a new entry (off by default). Backends - Anthropic prompt caching of the per-session system prompt (llm.prompt_cache, default on; billing only). prompt_tokens stays the total input; cache writes and reads are reported separately. - UsageMeter counts every LLM call; totals are exported as journal llm_usage. - Models that reject `temperature` are retried once without it. - LLMResponse gains api_error, stop_reason, content_types and cache fields; AGENTIGRID_MAX_TOKENS overrides the output-token cap. UI and reports - Iterations shown as "N + base case" (summary_stats adds llm_iterations and has_base_case; total_iterations unchanged). - FAILED only for solves that did not converge, not for analysis / complete / sweep / contingency / explore entries or discarded proposals. - Sidebar checkbox for the stall breaker (unchecked by default). - Default Anthropic model claude-sonnet-4-20250514 (retired) -> claude-sonnet-4-6. Tests: new tests/llm/test_prompt_cache.py and tests/journal/test_iteration_count.py; 8 new tests in tests/agent_loop/test_agent_loop.py. Fast suite: 1103 passed, 12 skipped; the 2 failures are the existing OPFLOW fixture issues fixed by the genfuel PR. Co-Authored-By: Claude Opus 5.5 --- agentigrid/README.md | 8 +- agentigrid/agentigrid/backends/__init__.py | 4 +- .../agentigrid/backends/anthropic_backend.py | 79 ++++- agentigrid/agentigrid/backends/base.py | 72 +++++ .../agentigrid/backends/ollama_backend.py | 1 + .../agentigrid/backends/openai_backend.py | 1 + agentigrid/agentigrid/config.py | 15 +- agentigrid/agentigrid/engine/agent_loop.py | 291 +++++++++++++++--- agentigrid/agentigrid/engine/journal.py | 49 ++- .../agentigrid/engine/regenerate_report.py | 1 + agentigrid/agentigrid/engine/session_io.py | 1 + agentigrid/claude_code_changes.md | 34 ++ .../configs/default_config.yaml.template | 3 +- agentigrid/configs/local_config.yaml.template | 1 + agentigrid/launcher/ARCHITECTURE.md | 4 +- agentigrid/launcher/app.py | 40 ++- agentigrid/launcher/config_builder.py | 2 +- agentigrid/launcher/report_generator.py | 15 +- .../tests/agent_loop/test_agent_loop.py | 146 +++++++++ agentigrid/tests/e2e/test_e2e.py | 2 +- .../tests/journal/test_iteration_count.py | 51 +++ agentigrid/tests/llm/test_backends.py | 2 +- agentigrid/tests/llm/test_prompt_cache.py | 229 ++++++++++++++ 23 files changed, 985 insertions(+), 66 deletions(-) create mode 100644 agentigrid/tests/journal/test_iteration_count.py create mode 100644 agentigrid/tests/llm/test_prompt_cache.py diff --git a/agentigrid/README.md b/agentigrid/README.md index a731bec0..ced35f1a 100644 --- a/agentigrid/README.md +++ b/agentigrid/README.md @@ -307,7 +307,7 @@ agentigrid ./data/exago/examples/case_ACTIVSg200.m "Find the maximum load scalin # With options agentigrid ./data/exago/examples/case_ACTIVSg200.m "Minimize generation cost" \ - --backend anthropic --model claude-sonnet-4-20250514 \ + --backend anthropic --model claude-sonnet-4-6 \ --app opflow --max-iter 30 --verbose # Quiet mode (only show final summary) @@ -364,7 +364,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r Version 0.1.0 ============================================================ Backend: anthropic - Model: claude-sonnet-4-20250514 + Model: claude-sonnet-4-6 Application: opflow Base case: data/exago/examples/case_ACTIVSg200.m Goal: Find the maximum load scaling factor @@ -375,7 +375,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r [Iter 0] Running base case simulation... [Iter 0] Base case: CONVERGED, cost=$27,557.57 -[Iter 1] Sending prompt to anthropic (claude-sonnet-4-20250514)... +[Iter 1] Sending prompt to anthropic (claude-sonnet-4-6)... [Iter 1] LLM action: modify — "Scale all loads +20%" [Iter 1] Applied 1 command(s), 0 skipped [Iter 1] Simulation completed in 0.04s — CONVERGED, cost=$33,019.55 @@ -390,7 +390,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r ============================================================ Goal: Find the maximum load scaling factor Application: opflow - Backend: anthropic (claude-sonnet-4-20250514) + Backend: anthropic (claude-sonnet-4-6) Iterations: 6 (of max 20) Duration: 18.3 seconds Tokens used: ~12,450 (prompt: 9,200, completion: 3,250) diff --git a/agentigrid/agentigrid/backends/__init__.py b/agentigrid/agentigrid/backends/__init__.py index dd3651b0..15114dfa 100644 --- a/agentigrid/agentigrid/backends/__init__.py +++ b/agentigrid/agentigrid/backends/__init__.py @@ -2,10 +2,10 @@ from __future__ import annotations -from agentigrid.backends.base import LLMBackend, LLMResponse +from agentigrid.backends.base import LLMBackend, LLMResponse, UsageMeter from agentigrid.config import LLMConfig -__all__ = ["LLMBackend", "LLMResponse", "create_backend"] +__all__ = ["LLMBackend", "LLMResponse", "UsageMeter", "create_backend"] def create_backend(config: LLMConfig) -> LLMBackend: diff --git a/agentigrid/agentigrid/backends/anthropic_backend.py b/agentigrid/agentigrid/backends/anthropic_backend.py index 10bbea96..594ba46d 100644 --- a/agentigrid/agentigrid/backends/anthropic_backend.py +++ b/agentigrid/agentigrid/backends/anthropic_backend.py @@ -27,6 +27,9 @@ def __init__(self, config: LLMConfig) -> None: raise ImportError("anthropic package is required for the Anthropic backend") self._config = config + # None = not yet known; True/False once a call has succeeded / the + # model has rejected the parameter (see complete()). + self.temperature_sent: Optional[bool] = None api_key = os.environ.get(config.api_key_env) if not api_key: logger.warning( @@ -40,6 +43,26 @@ def name(self) -> str: """Return the backend name.""" return "anthropic" + def _system_param(self, system_prompt: str): + """System prompt in the form the Messages API expects. + + With ``prompt_cache`` on, the system prompt is sent as a single text block + carrying an ephemeral ``cache_control`` breakpoint. AgentiGrid builds the + system prompt once per session, so every later iteration reads it from the + cache instead of paying the full input price again. This affects billing + only; the model sees identical text either way. Prompts below the model's + minimum cacheable length are silently processed uncached by the API. + """ + if not getattr(self._config, "prompt_cache", False): + return system_prompt + return [ + { + "type": "text", + "text": system_prompt, + "cache_control": {"type": "ephemeral"}, + } + ] + def supports_json_mode(self) -> bool: """Anthropic does not have a native JSON output mode.""" return False @@ -54,20 +77,45 @@ def complete( temp = temperature if temperature is not None else self._config.temperature try: - response = self._client.messages.create( + kwargs = dict( model=self._config.model, max_tokens=self._config.max_tokens, - temperature=temp, - system=system_prompt, + system=self._system_param(system_prompt), messages=[{"role": "user", "content": user_prompt}], ) + if self.temperature_sent is not False: + kwargs["temperature"] = temp + try: + response = self._client.messages.create(**kwargs) + if "temperature" in kwargs: + self.temperature_sent = True + except Exception as exc: + # Newer models reject the sampling-temperature parameter + # ("`temperature` is deprecated for this model"). Retry once + # without it and stop sending it for the rest of the session; + # the journal records temperature_sent=False. + if "temperature" in kwargs and "temperature" in str(exc).lower(): + logger.warning("Model %s rejects `temperature`; retrying without it.", self._config.model) + kwargs.pop("temperature") + self.temperature_sent = False + response = self._client.messages.create(**kwargs) + else: + raise raw_text = "" + content_types = [] for block in response.content: + content_types.append(getattr(block, "type", None)) if block.type == "text": raw_text += block.text + stop_reason = getattr(response, "stop_reason", None) + if stop_reason == "max_tokens": + logger.warning("Response stopped at max_tokens=%s (blocks: %s, %d text chars)", + self._config.max_tokens, content_types, len(raw_text)) - prompt_tokens = getattr(response.usage, "input_tokens", None) + prompt_tokens, cache_creation, cache_read = _input_token_breakdown( + response.usage + ) completion_tokens = getattr(response.usage, "output_tokens", None) json_data, json_error = extract_json(raw_text) @@ -80,6 +128,10 @@ def complete( backend=self.name(), prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, + stop_reason=stop_reason, + content_types=content_types, ) except Exception as exc: @@ -93,4 +145,23 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) + + +def _input_token_breakdown(usage) -> tuple[Optional[int], Optional[int], Optional[int]]: + """Return (total_input, cache_creation, cache_read) from an API usage object. + + With caching, the API's ``input_tokens`` counts only tokens after the last + cache breakpoint. The total input is ``input_tokens + cache_creation_input_tokens + + cache_read_input_tokens``; returning that total keeps ``prompt_tokens`` + comparable with runs made before caching was enabled. + """ + base = getattr(usage, "input_tokens", None) + creation = getattr(usage, "cache_creation_input_tokens", None) + read = getattr(usage, "cache_read_input_tokens", None) + creation = creation if isinstance(creation, int) else None + read = read if isinstance(read, int) else None + if not isinstance(base, int): + return None, creation, read + return base + (creation or 0) + (read or 0), creation, read diff --git a/agentigrid/agentigrid/backends/base.py b/agentigrid/agentigrid/backends/base.py index 32c4581a..2e681924 100644 --- a/agentigrid/agentigrid/backends/base.py +++ b/agentigrid/agentigrid/backends/base.py @@ -18,6 +18,20 @@ class LLMResponse: backend: str prompt_tokens: Optional[int] completion_tokens: Optional[int] + # Prompt-cache accounting (Anthropic). ``prompt_tokens`` always reports the + # TOTAL input tokens (uncached + cache writes + cache reads), so its meaning + # is the same with or without caching; these break that total down. + cache_creation_tokens: Optional[int] = None + cache_read_tokens: Optional[int] = None + # True when the request itself failed (network, auth, invalid request...), + # as opposed to the model answering with unparseable text. Lets the agent + # loop record API failures separately from rejected proposals. + api_error: bool = False + # Why generation stopped ("end_turn", "max_tokens", ...) and the content + # block types returned, when the backend reports them. A "max_tokens" stop + # is an output-cap limit, not a model proposal, and is recorded as such. + stop_reason: Optional[str] = None + content_types: Optional[list] = None class LLMBackend(ABC): @@ -51,3 +65,61 @@ def name(self) -> str: def supports_json_mode(self) -> bool: """Whether this backend supports a native JSON output mode.""" ... + + +class UsageMeter(LLMBackend): + """Transparent wrapper that meters token usage across EVERY backend call. + + The agent loop's own counters cover only the main per-iteration call; the + post-search analysis calls are not counted there. Wrapping the backend once + gives complete per-run totals (including prompt-cache writes and reads) for + cost accounting, without touching any call site. Behaviour is unchanged: + every call is delegated verbatim. + """ + + def __init__(self, inner: LLMBackend) -> None: + self._inner = inner + self.calls = 0 + self.prompt_tokens = 0 + self.completion_tokens = 0 + self.cache_creation_tokens = 0 + self.cache_read_tokens = 0 + self.api_errors = 0 + self.truncated = 0 + + def complete(self, system_prompt, user_prompt, temperature=None): + resp = self._inner.complete(system_prompt, user_prompt, temperature) + self.calls += 1 + if getattr(resp, "api_error", False): + self.api_errors += 1 + if getattr(resp, "stop_reason", None) == "max_tokens": + self.truncated += 1 + self.prompt_tokens += resp.prompt_tokens or 0 + self.completion_tokens += resp.completion_tokens or 0 + self.cache_creation_tokens += resp.cache_creation_tokens or 0 + self.cache_read_tokens += resp.cache_read_tokens or 0 + return resp + + def name(self) -> str: + return self._inner.name() + + def supports_json_mode(self) -> bool: + return self._inner.supports_json_mode() + + def __getattr__(self, item): # delegate anything backend-specific + if item == "_inner": # not yet set (e.g. during copy/unpickle) + raise AttributeError(item) + return getattr(self._inner, item) + + def totals(self) -> dict: + """Per-run usage totals, as written to the journal (``llm_usage``).""" + return { + "calls": self.calls, + "prompt_tokens": self.prompt_tokens, + "completion_tokens": self.completion_tokens, + "cache_creation_tokens": self.cache_creation_tokens, + "cache_read_tokens": self.cache_read_tokens, + "api_errors": self.api_errors, + "truncated": self.truncated, + "temperature_sent": getattr(self._inner, "temperature_sent", None), + } diff --git a/agentigrid/agentigrid/backends/ollama_backend.py b/agentigrid/agentigrid/backends/ollama_backend.py index f7fb1541..ae9faf6b 100644 --- a/agentigrid/agentigrid/backends/ollama_backend.py +++ b/agentigrid/agentigrid/backends/ollama_backend.py @@ -120,4 +120,5 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) diff --git a/agentigrid/agentigrid/backends/openai_backend.py b/agentigrid/agentigrid/backends/openai_backend.py index 0b4261ab..47ecb8f0 100644 --- a/agentigrid/agentigrid/backends/openai_backend.py +++ b/agentigrid/agentigrid/backends/openai_backend.py @@ -104,4 +104,5 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) diff --git a/agentigrid/agentigrid/config.py b/agentigrid/agentigrid/config.py index bca2bade..9eff56ea 100644 --- a/agentigrid/agentigrid/config.py +++ b/agentigrid/agentigrid/config.py @@ -35,13 +35,14 @@ }, "llm": { "backend": "anthropic", - "model": "claude-sonnet-4-20250514", + "model": "claude-sonnet-4-6", "api_key_env": "ANTHROPIC_API_KEY", "openai_base_url": None, "ollama_host": "http://localhost:11434", "ollama_cloud_host": None, "temperature": 0.3, "max_tokens": 4096, + "prompt_cache": True, }, "search": { "max_iterations": 20, @@ -132,6 +133,9 @@ class LLMConfig: ollama_cloud_host: Optional[str] temperature: float max_tokens: int + # Anthropic prompt caching of the (per-session constant) system prompt. + # Billing-only: does not change model outputs. Ignored by other backends. + prompt_cache: bool = True @dataclass(frozen=True) @@ -311,6 +315,15 @@ def load_config( target[parts[-1]] = value logger.debug("Applied CLI overrides: %s", cli_overrides) + # AGENTIGRID_MAX_TOKENS overrides the output-token cap (e.g. to fix it for a + # batch of runs); the value used is recorded in the journal's llm_usage. + _mt = os.environ.get("AGENTIGRID_MAX_TOKENS", "").strip() + if _mt: + try: + merged["llm"] = {**merged["llm"], "max_tokens": int(_mt)} + except ValueError: + logger.warning("Ignoring non-integer AGENTIGRID_MAX_TOKENS=%r", _mt) + # Build frozen dataclass sections exago_path_fields = { "binary_dir", "opflow_binary", "scopflow_binary", "tcopflow_binary", diff --git a/agentigrid/agentigrid/engine/agent_loop.py b/agentigrid/agentigrid/engine/agent_loop.py index 23dae310..37b744a8 100644 --- a/agentigrid/agentigrid/engine/agent_loop.py +++ b/agentigrid/agentigrid/engine/agent_loop.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Callable, Optional -from agentigrid.backends import create_backend +from agentigrid.backends import UsageMeter, create_backend from agentigrid.backends.base import LLMBackend, LLMResponse from agentigrid.config import AppConfig from agentigrid.engine.commands import parse_command @@ -30,7 +30,7 @@ compute_pareto_labels, format_variant_results, ) -from agentigrid.engine.journal import JournalEntry, ObjectiveEntry, SearchJournal +from agentigrid.engine.journal import JournalEntry, ObjectiveEntry, SearchJournal, format_iteration_count from agentigrid.engine.metric_extractor import available_metrics, available_metrics_for_app, extract_all_metrics from agentigrid.engine.modifier import apply_modifications, build_index_maps from agentigrid.engine import sweep_metrics @@ -66,6 +66,9 @@ def _count_scenario_rows(scenario_path: Path) -> int: logger = logging.getLogger("agentigrid.engine.agent_loop") +# Cap on the raw model response kept for a discarded iteration (journal telemetry). +DISCARDED_RESPONSE_MAX = 4000 + _MAX_CONSECUTIVE_PARSE_FAILURES = 3 @@ -765,7 +768,9 @@ def __init__( on_explore: Callable[[int, list[dict]], None] | None = None, ) -> None: self._config = config - self._backend: LLMBackend = create_backend(config.llm) + # Metered so per-run token totals (incl. prompt-cache reads/writes) + # cover every LLM call, not only the main per-iteration call. + self._backend: LLMBackend = UsageMeter(create_backend(config.llm)) self._executor = SimulationExecutor(config.exago, config.output) self._journal = SearchJournal() self._quiet = quiet @@ -1217,17 +1222,67 @@ def run(self, base_case: Path, goal: str) -> SearchSession: # 3. Agent loop max_iter = self._config.search.max_iterations + # Optional stall breaker: stop after N consecutive no-progress iterations + # (iterations that record no new journal entry — e.g. the model repeating + # an invalid action). 0/unset disables it (the default), so batch runs always + # reach max_iter and stay comparable; the UI can set it. + try: + stall_limit = int(os.environ.get("AGENTIGRID_STALL_LIMIT", "0") or "0") + except ValueError: + stall_limit = 0 + stall = 0 for iteration in range(1, max_iter + 1): if self._stop_requested: session.termination_reason = "user_stopped" self._print("\nSearch stopped by user.") break - action_type, should_continue = self._iteration(iteration, goal) - # Notify callback after each iteration + self._iteration_api_error = False + self._iteration_response_text = None + self._iteration_stop_reason = None + self._iteration_content_types = None + try: + action_type, should_continue = self._iteration(iteration, goal) + except Exception as e: + import traceback + self._print( + f"\n[Iter {iteration}] Action failed with an internal error " + f"({type(e).__name__}: {e}); discarding and continuing." + ) + traceback.print_exc() + self._error_feedback = ( + f"The previous action raised an internal error and was discarded: " + f"{type(e).__name__}: {e}. Respond with a well-formed action that " + f"strictly follows the schema." + ) + action_type, should_continue = "error", True + internal_error = True + else: + internal_error = False + # Did THIS iteration record a journal entry? A discarded/invalid action + # (common with weak local models) appends nothing. + latest_entry = self._journal.latest + recorded = latest_entry is not None and latest_entry.iteration == iteration + if action_type == "error" and not recorded: + self._record_discarded(iteration, internal_error) + # Notify the UI only on a real new entry; otherwise show a transient + # discarded card (avoids re-emitting the previous entry as a phantom). if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: + if recorded: self._on_iteration(iteration, latest_entry, action_type, self._latest_opflow) + elif action_type == "error": + self._emit_discarded(iteration) + # Stall breaker: count consecutive no-progress iterations. + if recorded: + stall = 0 + elif stall_limit: + stall += 1 + if stall >= stall_limit: + session.termination_reason = "stalled_no_progress" + self._print( + f"\nStopping early: {stall} consecutive no-progress " + f"iterations (stall limit {stall_limit})." + ) + break if not should_continue: if not session.termination_reason: session.termination_reason = "completed" @@ -1239,16 +1294,12 @@ def run(self, base_case: Path, goal: str) -> SearchSession: if not session.termination_reason: session.termination_reason = "completed" - # Final notification with termination reason - if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: - self._on_iteration( - latest_entry.iteration, - latest_entry, - session.termination_reason, - self._latest_opflow, - ) + # No final on_iteration re-emit: the last real entry was already sent by + # the per-iteration callback above. Re-emitting it duplicated the last + # timeline card, and when the final iterations were discarded it made a + # stale, lower-numbered entry appear AFTER the final iteration (e.g. a + # phantom "Iteration 6" after 20). The UI reads session.termination_reason + # when the search finishes. session.end_time = datetime.now().isoformat() session.total_prompt_tokens = self._total_prompt_tokens @@ -1266,6 +1317,88 @@ def run(self, base_case: Path, goal: str) -> SearchSession: self._finalize(session, elapsed) return session + # ------------------------------------------------------------------ + # Discarded-iteration diagnostic + # ------------------------------------------------------------------ + + def _record_discarded(self, iteration: int, internal: bool = False) -> None: + """Journal telemetry for an iteration whose action produced no entry. + + Persisted: iteration, kind (rejected / internal / api_error / + truncated), stop_reason and content block types when known, the + truncated feedback sent back to the model and, when the model answered, + its raw response (the rejected proposal itself, capped at + DISCARDED_RESPONSE_MAX characters). The entry list and all summary + statistics are unaffected.""" + stop = getattr(self, "_iteration_stop_reason", None) + if internal: + kind = "internal" + elif getattr(self, "_iteration_api_error", False): + kind = "api_error" + elif stop == "max_tokens": + kind = "truncated" # output cap hit: a harness limit, not a rejected proposal + else: + kind = "rejected" + record = { + "iteration": iteration, + "kind": kind, + "feedback": (self._error_feedback or "")[:300], + } + if stop is not None: + record["stop_reason"] = stop + ctypes = getattr(self, "_iteration_content_types", None) + if ctypes is not None: + record["content_types"] = list(ctypes) + text = getattr(self, "_iteration_response_text", None) + if text is not None and kind != "api_error": # an API error has no model response + record["response"] = text[:DISCARDED_RESPONSE_MAX] + record["response_truncated"] = len(text) > DISCARDED_RESPONSE_MAX + self._journal.discarded_actions.append(record) + + def _emit_discarded(self, iteration: int) -> None: + """Emit a UI-only 'discarded' timeline card for an iteration whose action + was rejected (invalid/malformed proposal, unknown action, parse error) + and therefore recorded no journal entry. + + The synthetic entry is deliberately NOT added to ``self._journal``: it + never affects summary stats, best-iteration selection, or the persisted + record. ``mode="discarded"`` keeps ``is_solve_iteration`` + False, and ``feasible=False`` / ``objective_value=None`` keep it out of + the session_manager's best-cost tracking. It exists purely to tell the + user *why* the iteration produced nothing, instead of showing a phantom + duplicate or a blank card. + """ + if not self._on_iteration: + return + reason = (self._error_feedback or "The model returned an invalid proposal.").strip() + # The UI label must attribute the failure to the MODEL, not the code: + # the raw feedback ("Unknown action X. Valid actions: ...") is written for + # the LLM and reads like a code assertion in the timeline. Keep only the + # first clause for the headline and prefix it clearly; the full corrective + # text still shows in the card's expander (llm_reasoning). + headline = reason.split(". ")[0].splitlines()[0].strip().rstrip(".") + if not headline: + headline = "invalid proposal" + headline = headline if len(headline) <= 120 else headline[:117] + "..." + synthetic = JournalEntry( + iteration=iteration, + description=f"LLM output rejected — {headline} (no change applied)", + commands=[], + objective_value=None, + feasible=False, + convergence_status="FAILED", + violations_count=0, + voltage_min=0.0, + voltage_max=0.0, + max_line_loading_pct=0.0, + total_gen_mw=0.0, + total_load_mw=0.0, + llm_reasoning=reason, # full detail shown in the card's expander + mode="discarded", + elapsed_seconds=0.0, + ) + self._on_iteration(iteration, synthetic, "discarded", None) + # ------------------------------------------------------------------ # Single iteration # ------------------------------------------------------------------ @@ -1322,6 +1455,11 @@ def _iteration( response = self._backend.complete(system_prompt, user_prompt) logger.debug("LLM raw response: %s", response.raw_text[:500]) + # Kept for journal telemetry if this iteration's action is discarded: + # the exact proposal the parser / validator rejected. + self._iteration_response_text = response.raw_text or "" + self._iteration_stop_reason = getattr(response, "stop_reason", None) + self._iteration_content_types = getattr(response, "content_types", None) # Track tokens pt = response.prompt_tokens or 0 @@ -1334,6 +1472,11 @@ def _iteration( f"(cumulative: ~{self._total_prompt_tokens + self._total_completion_tokens:,})" ) + # A failed request (auth, invalid request, network) is not a model + # answer: journal it as an API error, not as a rejected action. + if getattr(response, "api_error", False): + self._iteration_api_error = True + # Parse JSON from response if response.json_data is None: self._consecutive_parse_failures += 1 @@ -1343,14 +1486,23 @@ def _iteration( _MAX_CONSECUTIVE_PARSE_FAILURES, response.json_error, ) - self._print(f"[Iter {iteration}] Failed to parse LLM response as JSON") + truncated = getattr(response, "stop_reason", None) == "max_tokens" + self._print(f"[Iter {iteration}] Failed to parse LLM response as JSON" + + (f" (output hit max_tokens={self._config.llm.max_tokens})" if truncated else "")) if self._consecutive_parse_failures >= _MAX_CONSECUTIVE_PARSE_FAILURES: self._print(f"[Iter {iteration}] Too many consecutive parse failures — aborting") return "error", False - self._error_feedback = ( - "Failed to parse JSON from your response. " - "Please respond with a valid JSON object." - ) + if truncated: + self._error_feedback = ( + "Your previous response was cut off at the output-token limit before " + "a complete JSON object was produced. Respond with the JSON action " + "only, keeping the reasoning brief." + ) + else: + self._error_feedback = ( + "Failed to parse JSON from your response. " + "Please respond with a valid JSON object." + ) return "error", True self._consecutive_parse_failures = 0 @@ -1477,6 +1629,14 @@ def _handle_modify( f"{skipped_count} skipped" ) + # A modify that applied zero commands changes nothing — make the timeline + # say so instead of showing a blank "No description" card. + if applied_count == 0: + if not description or description == "No description": + description = "No commands applied (no change)" + else: + description = f"{description} — no commands applied" + if all_errors: self._error_feedback = "Command errors:\n" + "\n".join(all_errors) @@ -1592,7 +1752,7 @@ def _handle_modify( self._active_steering_directives[-1]["directive"] if self._active_steering_directives else None ) - self._journal.add_from_results( + _modify_entry = self._journal.add_from_results( iteration=iteration, description=description, commands=raw_commands, @@ -1606,6 +1766,7 @@ def _handle_modify( gencost=modified_net.gencost if self._config.search.application == "pflow" else None, exago_command=_single_call_record(sim_result), ) + _modify_entry.skipped_commands = list(all_errors) # Extract tracked metrics for multi-objective tracking if opflow is not None: @@ -2043,6 +2204,11 @@ def _resolve_candidate_buses(self, spec: dict) -> tuple[Optional[list[int]], str Returns (candidates, "") on success or (None, error_message) on failure. Shared by the feasibility sweep and the boundary sweep. """ + if not isinstance(spec, dict): + return None, ( + f"candidate_set must be an object with a 'type' field, " + f"got {type(spec).__name__}: {spec!r}" + ) ctype = spec.get("type", "") if ctype == "all_buses": candidates = [b.bus_i for b in self._base_network.buses] @@ -2200,6 +2366,8 @@ def _handle_sweep( candidate_set_spec = data.get("candidate_set", {}) mutation_template = data.get("mutation", {}) feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} if not mutation_template or "action" not in mutation_template: self._error_feedback = "sweep requires a 'mutation' dict with an 'action' key." @@ -2624,6 +2792,8 @@ def _handle_boundary_sweep( pf_spec = data.get("power_factor", s.boundary_power_factor_default) candidate_set_spec = data.get("candidate_set", {}) feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} vmin = feasibility_spec.get("Vmin", 0.9) vmax = feasibility_spec.get("Vmax", 1.1) q_frac = s.boundary_gen_q_frac @@ -3228,6 +3398,8 @@ def _handle_reserve_screen( description = data.get("description", "Hot reserve / N-1 generator security") reasoning = data.get("reasoning", "") feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} vmin = feasibility_spec.get("Vmin", 0.9) vmax = feasibility_spec.get("Vmax", 1.1) minimize = bool(data.get("minimize", False)) @@ -4641,6 +4813,16 @@ def _assemble_prompt( def _finalize(self, session: SearchSession, elapsed_seconds: float) -> None: """Print summary and save journal.""" + _totals = getattr(self._backend, "totals", None) + if callable(_totals): + self._journal.llm_usage = { + **_totals(), + "backend": self._backend.name(), + "model": self._config.llm.model, + "max_tokens": self._config.llm.max_tokens, + "prompt_cache": bool(getattr(self._config.llm, "prompt_cache", False)), + } + total_tokens = self._total_prompt_tokens + self._total_completion_tokens # --- Post-search goal classification via LLM --- @@ -4699,7 +4881,7 @@ def _finalize(self, session: SearchSession, elapsed_seconds: float) -> None: print(f" Application: {session.application}") print(f" Backend: {self._backend.name()} ({self._config.llm.model})") print( - f" Iterations: {stats['total_iterations']} " + f" Iterations: {format_iteration_count(stats)} " f"(of max {self._config.search.max_iterations})" ) print(f" Duration: {elapsed_seconds:.1f} seconds") @@ -4968,16 +5150,61 @@ def resume_from(self, save_dir: Path) -> SearchSession: # Continue the agent loop from last_iteration + 1 max_iter = self._config.search.max_iterations + try: + stall_limit = int(os.environ.get("AGENTIGRID_STALL_LIMIT", "0") or "0") + except ValueError: + stall_limit = 0 + stall = 0 for iteration in range(last_iteration + 1, max_iter + 1): if self._stop_requested: session.termination_reason = "user_stopped" self._print("\nSearch stopped by user.") break - action_type, should_continue = self._iteration(iteration, goal) + self._iteration_api_error = False + self._iteration_response_text = None + self._iteration_stop_reason = None + self._iteration_content_types = None + try: + action_type, should_continue = self._iteration(iteration, goal) + except Exception as e: + import traceback + self._print( + f"\n[Iter {iteration}] Action failed with an internal error " + f"({type(e).__name__}: {e}); discarding and continuing." + ) + traceback.print_exc() + self._error_feedback = ( + f"The previous action raised an internal error and was discarded: " + f"{type(e).__name__}: {e}. Respond with a well-formed action that " + f"strictly follows the schema." + ) + action_type, should_continue = "error", True + internal_error = True + else: + internal_error = False + + # Only emit when this iteration recorded a new entry (see run() above). + latest_entry = self._journal.latest + recorded = latest_entry is not None and latest_entry.iteration == iteration + if action_type == "error" and not recorded: + self._record_discarded(iteration, internal_error) if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: + if recorded: self._on_iteration(iteration, latest_entry, action_type, self._latest_opflow) + elif action_type == "error": + self._emit_discarded(iteration) + # Stall breaker: count consecutive no-progress iterations. + if recorded: + stall = 0 + elif stall_limit: + stall += 1 + if stall >= stall_limit: + session.termination_reason = "stalled_no_progress" + self._print( + f"\nStopping early: {stall} consecutive no-progress " + f"iterations (stall limit {stall_limit})." + ) + break if not should_continue: if not session.termination_reason: session.termination_reason = "completed" @@ -4989,14 +5216,8 @@ def resume_from(self, save_dir: Path) -> SearchSession: if not session.termination_reason: session.termination_reason = "completed" - # Final notification - if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: - self._on_iteration( - latest_entry.iteration, latest_entry, - session.termination_reason, self._latest_opflow, - ) + # No final on_iteration re-emit (see run() above) — it only duplicated + # the last timeline card. session.end_time = datetime.now().isoformat() session.total_prompt_tokens = self._total_prompt_tokens diff --git a/agentigrid/agentigrid/engine/journal.py b/agentigrid/agentigrid/engine/journal.py index 4aa8a97a..0a248cb3 100644 --- a/agentigrid/agentigrid/engine/journal.py +++ b/agentigrid/agentigrid/engine/journal.py @@ -54,6 +54,10 @@ class JournalEntry: exago_command: Optional[dict] = None # Reproducible ExaGO invocation record (JSON journal only; see add_* methods) contingency_meta: Optional[dict] = None # C.5: target bus, neighbors, order, pass/fail counts reserve_meta: Optional[dict] = None # C.8: hot-reserve / N-1 generator security accounting + # Modify iterations: commands the LLM proposed that were NOT applied (parse + # errors or validator skips), as messages. [] = every proposed command was + # applied; None = not recorded (non-modify entries, older journals). + skipped_commands: Optional[list[str]] = None def is_solve_iteration(entry: JournalEntry) -> bool: @@ -187,6 +191,11 @@ def __init__(self) -> None: self.session_best: Optional[dict] = None # load_factor: session-level load scaling factor (updated by set_load_factor action). self.load_factor: Optional[float] = None + # discarded_actions: iterations whose LLM action recorded no entry -- + # rejected by the deterministic validator / parser ("rejected"), a failed + # API request ("api_error"), output cut at max_tokens ("truncated") or + # an internal error ("internal"). Exported with the journal. + self.discarded_actions: list[dict] = [] def update_session_best( self, @@ -1075,6 +1084,10 @@ def export_json(self, path: Path) -> None: data["session_best"] = self.session_best if self.load_factor is not None: data["load_factor"] = self.load_factor + if getattr(self, "llm_usage", None) is not None: + data["llm_usage"] = self.llm_usage + if getattr(self, "discarded_actions", None): + data["discarded_actions"] = self.discarded_actions path.write_text(json.dumps(data, indent=2), encoding="utf-8") logger.info("Journal exported to %s (%d entries)", path, len(self._entries)) @@ -1092,7 +1105,7 @@ def export_csv(self, path: Path) -> None: "mode", "elapsed_seconds", "timestamp", "steering_directive", "tracked_metrics", "feasibility_detail", "solver", "num_steps", "num_scenarios", "explored_variants", "candidate_count", "feasible_buses", "exago_command", - "contingency_meta", "reserve_meta", + "contingency_meta", "reserve_meta", "skipped_commands", ] with open(path, "w", newline="", encoding="utf-8") as f: @@ -1107,6 +1120,7 @@ def export_csv(self, path: Path) -> None: row["exago_command"] = json.dumps(row.get("exago_command") or None) row["contingency_meta"] = json.dumps(row.get("contingency_meta") or None) row["reserve_meta"] = json.dumps(row.get("reserve_meta") or None) + row["skipped_commands"] = json.dumps(row.get("skipped_commands")) writer.writerow(row) logger.info("Journal CSV exported to %s (%d entries)", path, len(self._entries)) @@ -1115,6 +1129,22 @@ def export_csv(self, path: Path) -> None: # Summary statistics # ------------------------------------------------------------------ + def _iteration_counts(self) -> tuple[int, bool]: + """Agent iterations actually run, and whether a base case was solved. + + ``total_iterations`` counts journal entries: it includes the base-case + solve (iteration 0) and leaves out iterations whose LLM output was + rejected (they are kept in ``discarded_actions``, not as entries). + This counts distinct iteration numbers >= 1 across both, which is the + number of agent iterations the user asked for and the run performed. + """ + numbers = {e.iteration for e in self._entries} + for d in getattr(self, "discarded_actions", None) or []: + it = d.get("iteration") if isinstance(d, dict) else None + if isinstance(it, int): + numbers.add(it) + return sum(1 for n in numbers if n >= 1), 0 in numbers + def summary_stats( self, best_iteration_override: int | None = None, @@ -1134,9 +1164,12 @@ def summary_stats( feasible_count, infeasible_count, objective_trend, voltage_range_trend, goal_type. """ + llm_iterations, has_base_case = self._iteration_counts() if not self._entries: return { "total_iterations": 0, + "llm_iterations": llm_iterations, + "has_base_case": has_base_case, "best_objective": None, "best_iteration": None, "best_bus": None, @@ -1194,6 +1227,8 @@ def summary_stats( return { "total_iterations": len(self._entries), + "llm_iterations": llm_iterations, + "has_base_case": has_base_case, "best_objective": best_objective, "best_iteration": best_iteration, "best_bus": best_bus, @@ -1248,3 +1283,15 @@ def _best_cost_in_sweeps(self) -> tuple[float, int, int | None] | None: if best is None or c < best[0]: best = (c, e.iteration, v.get("bus")) return best + + +def format_iteration_count(stats: dict) -> str: + """Display form of the iteration count, e.g. ``"2 + base case"``. + + Falls back to ``total_iterations`` for stats dicts written before + ``llm_iterations`` existed. + """ + if "llm_iterations" not in stats: + return str(stats.get("total_iterations", 0)) + n = stats["llm_iterations"] + return f"{n} + base case" if stats.get("has_base_case") else str(n) diff --git a/agentigrid/agentigrid/engine/regenerate_report.py b/agentigrid/agentigrid/engine/regenerate_report.py index 43cbd790..b456f024 100644 --- a/agentigrid/agentigrid/engine/regenerate_report.py +++ b/agentigrid/agentigrid/engine/regenerate_report.py @@ -110,6 +110,7 @@ def load_journal_export(path: Path) -> SearchJournal: exago_command=ed.get("exago_command"), contingency_meta=ed.get("contingency_meta"), reserve_meta=ed.get("reserve_meta"), + skipped_commands=ed.get("skipped_commands"), )) # Rebuild the objective registry, then overwrite its history with the saved diff --git a/agentigrid/agentigrid/engine/session_io.py b/agentigrid/agentigrid/engine/session_io.py index a5b3fe51..7971d4eb 100644 --- a/agentigrid/agentigrid/engine/session_io.py +++ b/agentigrid/agentigrid/engine/session_io.py @@ -184,6 +184,7 @@ def load_session(save_dir: Path) -> dict[str, Any]: exago_command=entry_data.get("exago_command"), contingency_meta=entry_data.get("contingency_meta"), reserve_meta=entry_data.get("reserve_meta"), + skipped_commands=entry_data.get("skipped_commands"), ) journal_entries.append(entry) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 29513d3d..f7a41952 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -58,6 +58,40 @@ Fast suite (`python -m pytest -m "not slow"`), without ExaGO binaries: **1083 pa --- +## Agent loop and LLM backends: robust iterations, honest UI, usage accounting (2026-10-03) + +**Motivation.** Long runs with local models (Ollama) showed iterations that went wrong in ways the UI hid or misreported: a malformed action crashed the run; a rejected iteration re-emitted the previous card, so the timeline showed stale or out-of-order cards (e.g. "Iteration 6" after iteration 20); analysis and completion entries were labelled FAILED; a modify that applied no command showed "No description"; and the summary counted the base case as an iteration ("Iterations 3" for max 2). Separately, Anthropic runs re-sent the same system prompt at full price on every iteration, and some newer models reject the `temperature` parameter. + +### Agent loop (`agentigrid/engine/agent_loop.py`) + +- Each iteration runs inside a guard: an exception in an action handler is discarded with feedback to the model, and the run continues. Handlers that read `feasibility` / `candidate_set` tolerate non-dict values. +- The UI callback fires only when the iteration recorded a journal entry. Otherwise a UI-only "discarded" card explains why ("LLM output rejected — … (no change applied)"); it is never added to the journal. The final re-emit of the last entry is gone (it duplicated the last card). Same in `resume`. +- `journal.discarded_actions` records every iteration that produced no entry: kind (`rejected`, `api_error`, `truncated`, `internal`), the feedback sent to the model, `stop_reason`, content block types and the raw response (capped at 4,000 characters). Entries and summary statistics are unchanged. +- A response cut at `max_tokens` gets its own message and feedback asking for the JSON action only. +- A modify that applies zero commands says so in its description; modify entries record `skipped_commands`. +- Optional stall breaker: `AGENTIGRID_STALL_LIMIT=N` ends the run after N consecutive iterations without a new entry (`termination_reason = "stalled_no_progress"`). Off by default. + +### Backends (`agentigrid/backends/`) + +- Anthropic prompt caching of the per-session system prompt (`llm.prompt_cache`, default on). Billing only; the model sees the same text. `prompt_tokens` stays the total input; cache writes and reads are reported separately. +- `UsageMeter` wraps the backend and counts every call (including post-search analysis); totals go to the journal as `llm_usage`. +- Models that reject `temperature` are retried once without it, and it is not sent again in that session. +- `LLMResponse` gains `api_error`, `stop_reason`, `content_types` and cache token fields; failed requests in all backends set `api_error`. +- `AGENTIGRID_MAX_TOKENS` overrides the output-token cap. + +### UI and reports + +- Iterations shown as "N + base case" (UI summary and live panel, CLI summary, PDF report). `summary_stats()` adds `llm_iterations` and `has_base_case`; `total_iterations` is unchanged. +- Timeline cards show FAILED only for solves that did not converge, not for analysis / complete / sweep / contingency / explore entries or discarded proposals. +- Sidebar checkbox for the stall breaker (unchecked by default). +- Default Anthropic model `claude-sonnet-4-20250514` (retired) → `claude-sonnet-4-6`. + +### Tests + +`tests/llm/test_prompt_cache.py` (new: caching, token breakdown, `UsageMeter`, temperature retry, API errors, max_tokens, env override), `tests/journal/test_iteration_count.py` (new), `tests/agent_loop/test_agent_loop.py` (+8: discarded telemetry, API errors, truncation, response cap, no stale UI cards, stall breaker off/on, internal errors). The 4 new agent-loop callback/stall/error tests fail on the previous agent loop. + +--- + ## C.2/C.3 corrections — metric gating, summary aggregation, predicate audit, sweep dedup (2026-06-23) Four contained fixes from the prompt-13/15/17 runs. No capability behavior changed (capacity numbers, dispatch results, metric/predicate definitions, and the C.1 binding identifier are untouched) — only how results are gated, summarized, audited, and deduplicated. diff --git a/agentigrid/configs/default_config.yaml.template b/agentigrid/configs/default_config.yaml.template index 9ef1edfc..d076872d 100644 --- a/agentigrid/configs/default_config.yaml.template +++ b/agentigrid/configs/default_config.yaml.template @@ -23,7 +23,7 @@ data: # LLM backend settings llm: backend: "anthropic" # openai | anthropic | ollama | ollama-cloud - model: "claude-sonnet-4-20250514" + model: "claude-sonnet-4-6" api_key_env: "ANTHROPIC_API_KEY" # env var name containing the API key # OpenAI-specific openai_base_url: null @@ -33,6 +33,7 @@ llm: # Generation parameters temperature: 0.3 max_tokens: 4096 + prompt_cache: true # Anthropic only: cache the system prompt (billing-only; outputs unchanged) # Search settings search: diff --git a/agentigrid/configs/local_config.yaml.template b/agentigrid/configs/local_config.yaml.template index 7442d60e..43566e21 100644 --- a/agentigrid/configs/local_config.yaml.template +++ b/agentigrid/configs/local_config.yaml.template @@ -22,6 +22,7 @@ llm: # ollama_cloud_host: null # set for Ollama Cloud temperature: 0.3 max_tokens: 4096 + prompt_cache: true # Anthropic only: cache the system prompt (billing-only; outputs unchanged) # Search settings search: diff --git a/agentigrid/launcher/ARCHITECTURE.md b/agentigrid/launcher/ARCHITECTURE.md index f8fa76d3..ecb9f7c9 100644 --- a/agentigrid/launcher/ARCHITECTURE.md +++ b/agentigrid/launcher/ARCHITECTURE.md @@ -224,7 +224,7 @@ Always visible. Contains all search parameters. **Section: LLM Backend** - Backend selector: `anthropic` | `openai` | `ollama` | `ollama-cloud` -- Model name text input (pre-filled with defaults per backend: `claude-sonnet-4-20250514` for Anthropic, `gpt-4o` for OpenAI, etc.) +- Model name text input (pre-filled with defaults per backend: `claude-sonnet-4-6` for Anthropic, `gpt-4o` for OpenAI, etc.) - Temperature slider (0.0 – 1.0, default 0.3) **Section: Search Parameters** @@ -274,7 +274,7 @@ Shown while a search is running. Two-column layout: - Key metrics: voltage range, max line loading, gen/load totals - Mode used (fresh/accumulative) - Current iteration shows a live status indicator: - - Spinner with phase text: "Sending prompt to Claude (claude-sonnet-4-20250514)..." → "Applying 3 modifications..." → "Running OPFLOW simulation..." → "Parsing results..." + - Spinner with phase text: "Sending prompt to Claude (claude-sonnet-4-6)..." → "Applying 3 modifications..." → "Running OPFLOW simulation..." → "Parsing results..." **Right column (~35%)** — Live Charts: - **Convergence chart** (Plotly line chart): Objective value vs. iteration number diff --git a/agentigrid/launcher/app.py b/agentigrid/launcher/app.py index 76339485..92fc4e05 100644 --- a/agentigrid/launcher/app.py +++ b/agentigrid/launcher/app.py @@ -37,6 +37,7 @@ ) from agentigrid.parsers import parse_matpower, network_summary +from agentigrid.engine.journal import NON_SOLVE_STATUSES, format_iteration_count # ── Page Configuration ─────────────────────────────────────────────────────── @@ -394,6 +395,23 @@ def render_sidebar() -> dict: max_iterations = st.slider( "Max iterations", 1, 50, 20, disabled=disabled, ) + stall_stop = st.checkbox( + "Stop early if the model stalls", + value=False, + disabled=disabled, + help=( + "End the search after N consecutive no-progress iterations " + "(the model repeating an invalid action or applying no change). " + "Saves iterations when a model is stuck. Leave unchecked for " + "comparable runs that always reach Max iterations." + ), + ) + stall_limit = st.number_input( + "Stop after N no-progress iterations", + min_value=2, max_value=20, value=5, step=1, + disabled=disabled or not stall_stop, + ) + os.environ["AGENTIGRID_STALL_LIMIT"] = str(int(stall_limit)) if stall_stop else "0" mpi_disabled = disabled or application not in ("scopflow", "sopflow") mpi_np = st.number_input( "MPI processes", @@ -797,15 +815,23 @@ def render_live_monitor(): icon = _iteration_icon(entry) obj = entry.get("objective_value") is_pflow_live = st.session_state.get("current_application") == "pflow" - if is_pflow_live: - obj_str = "analysis" if obj is not None else "FAILED" + # An entry with no objective is only "FAILED" if it was a real solve + # that did not converge. Analysis/complete/sweep/contingency/explore + # markers and discarded proposals carry no cost — they are not failures. + cs = entry.get("convergence_status") or "" + _no_solve = cs in NON_SOLVE_STATUSES or entry.get("mode") == "discarded" + if obj is not None: + obj_str = "analysis" if is_pflow_live else f"${obj:,.2f}" + elif _no_solve: + obj_str = "" # no cost, but not a failure — omit the segment else: - obj_str = f"${obj:,.2f}" if obj is not None else "FAILED" + obj_str = "FAILED" # a solve that genuinely did not converge elapsed = entry.get("sim_elapsed") time_str = f"{elapsed:.1f}s" if elapsed is not None else "—" + _cost_seg = f" — {obj_str}" if obj_str else "" label = ( f"{icon} Iteration {entry['iteration']}: " - f"{entry['description']} — {obj_str} ({time_str})" + f"{entry['description']}{_cost_seg} ({time_str})" ) with st.expander(label): @@ -933,7 +959,9 @@ def render_live_monitor(): # Progress Stats st.markdown("---") - n_iters = len(st.session_state.iteration_log) + _its = {e.get("iteration") for e in st.session_state.iteration_log} + _n_agent = sum(1 for i in _its if isinstance(i, int) and i >= 1) + n_iters = f"{_n_agent} + base case" if 0 in _its else str(_n_agent) feasible_count = sum( 1 for e in st.session_state.iteration_log if e.get("feasible") ) @@ -1329,7 +1357,7 @@ def _render_overview_tab(session): mc1, mc2, mc3, mc4, mc5 = st.columns(5) mc1.metric("Application", f"{session.application}") - mc2.metric("Iterations", f"{stats['total_iterations']}") + mc2.metric("Iterations", format_iteration_count(stats)) mc3.metric("Duration", duration_str) mc4.metric("Termination", session.termination_reason) mc5.metric("Tokens", f"{total_tokens:,}" if total_tokens > 0 else "—") diff --git a/agentigrid/launcher/config_builder.py b/agentigrid/launcher/config_builder.py index 4e13f207..a410bb61 100644 --- a/agentigrid/launcher/config_builder.py +++ b/agentigrid/launcher/config_builder.py @@ -253,7 +253,7 @@ def load_example_goals() -> list[dict[str, str]]: # Default model names per backend DEFAULT_MODELS: dict[str, str] = { - "anthropic": "claude-sonnet-4-20250514", + "anthropic": "claude-sonnet-4-6", "openai": "gpt-4o", "ollama": "qwen2.5:7b", "ollama-cloud": "qwen2.5:7b", diff --git a/agentigrid/launcher/report_generator.py b/agentigrid/launcher/report_generator.py index 22076181..d6a7f2e2 100644 --- a/agentigrid/launcher/report_generator.py +++ b/agentigrid/launcher/report_generator.py @@ -23,7 +23,7 @@ from reportlab.pdfbase.ttfonts import TTFont from agentigrid.engine.agent_loop import SearchSession -from agentigrid.engine.journal import is_solve_iteration +from agentigrid.engine.journal import format_iteration_count, is_solve_iteration from agentigrid.parsers.opflow_results import OPFLOWResult try: @@ -599,7 +599,7 @@ def _build_executive_summary( "is not deliverable for these losses (required reserve is mis-located)." ) res_lines.extend([ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", (f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A"), @@ -653,7 +653,7 @@ def _build_executive_summary( relief_line += f"; {unresolved_count} unresolved." cont_lines.append(relief_line) cont_lines.extend([ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", (f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A"), @@ -691,7 +691,7 @@ def _build_executive_summary( _headline = f"Sweep complete: {n_feas} of {n_cand} candidate buses are feasible." sweep_lines = [ _headline, - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A", @@ -715,7 +715,7 @@ def _build_executive_summary( 1 for e in solve_entries if e.feasibility_detail == "marginal" ) lines = [ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Feasible solutions: {solve_feasible}", f"Infeasible: {solve_infeasible}", ] @@ -1539,8 +1539,9 @@ def _build_convergence_section( ) # Auto-generated text - n = stats["total_iterations"] - text = f"The search ran for {n} iterations." + n = stats.get("llm_iterations", stats["total_iterations"]) + text = (f"The search ran for {n} iteration(s) after the base case." + if stats.get("has_base_case") else f"The search ran for {n} iteration(s).") if stats["best_objective"] is not None: base_entry = session.journal.entries[0] if session.journal.entries else None if base_entry and base_entry.objective_value and base_entry.objective_value != 0: diff --git a/agentigrid/tests/agent_loop/test_agent_loop.py b/agentigrid/tests/agent_loop/test_agent_loop.py index 2742af12..32222f4d 100644 --- a/agentigrid/tests/agent_loop/test_agent_loop.py +++ b/agentigrid/tests/agent_loop/test_agent_loop.py @@ -254,6 +254,152 @@ def test_invalid_json_then_valid(self, tmp_path: Path): assert len(session.journal) == 3 # base + modify + complete assert session.termination_reason == "completed" + def test_discarded_iterations_are_journaled_as_telemetry(self, tmp_path: Path): + """An unparseable action and an unknown action record no entry, but are + kept in journal.discarded_actions (validator-rejection telemetry).""" + cfg = _make_config(tmp_path, max_iterations=10) + responses = [ + _objectives_response(), # objective parser + _make_llm_response(None, raw_text="This is not JSON"), # iter 1 + _make_llm_response({"action": "dance", "reasoning": "x"}), # iter 2 + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}}), # iter 3 + ] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + controller = AgentLoopController(cfg) + session = controller.run(BASE_CASE, "Test discard telemetry") + + assert len(session.journal) == 2 # base + complete + disc = session.journal.discarded_actions + assert [(d["iteration"], d["kind"]) for d in disc] == [(1, "rejected"), (2, "rejected")] + assert "parse JSON" in disc[0]["feedback"] and "Unknown action" in disc[1]["feedback"] + # the rejected proposal itself is kept (raw model output) + assert disc[0]["response"] == "This is not JSON" and disc[0]["response_truncated"] is False + assert '"dance"' in disc[1]["response"] or "dance" in disc[1]["response"] + + def test_api_errors_are_not_counted_as_rejections(self, tmp_path: Path): + cfg = _make_config(tmp_path, max_iterations=10) + api_err = LLMResponse(raw_text="Anthropic API error: 400", json_data=None, json_error="x", + model="m", backend="test", prompt_tokens=None, completion_tokens=None, + api_error=True) + responses = [_objectives_response(), api_err, + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test api error") + disc = session.journal.discarded_actions + assert [(d["iteration"], d["kind"]) for d in disc] == [(1, "api_error")] + assert "response" not in disc[0] # no model response for a failed request + + def test_truncated_output_is_journaled_as_truncated(self, tmp_path: Path): + cfg = _make_config(tmp_path, max_iterations=10) + cut = LLMResponse(raw_text="", json_data=None, json_error="no JSON", model="m", + backend="test", prompt_tokens=10, completion_tokens=4096, + stop_reason="max_tokens", content_types=["thinking"]) + responses = [_objectives_response(), cut, + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test truncation") + d = session.journal.discarded_actions[0] + assert (d["iteration"], d["kind"], d["stop_reason"]) == (1, "truncated", "max_tokens") + assert d["content_types"] == ["thinking"] and d["response"] == "" + assert "output-token limit" in d["feedback"] + + def test_long_discarded_response_is_capped(self, tmp_path: Path): + from agentigrid.engine import agent_loop as al + cfg = _make_config(tmp_path, max_iterations=10) + long_text = "x" * (al.DISCARDED_RESPONSE_MAX + 50) + responses = [_objectives_response(), _make_llm_response(None, raw_text=long_text), + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test cap") + d = session.journal.discarded_actions[0] + assert len(d["response"]) == al.DISCARDED_RESPONSE_MAX and d["response_truncated"] is True + + def _run(self, tmp_path, responses, goal="Test", max_iterations=10, **ctl_kw): + cfg = _make_config(tmp_path, max_iterations=max_iterations) + mock_backend = MockBackend([_objectives_response()] + responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + return AgentLoopController(cfg, **ctl_kw).run(BASE_CASE, goal) + + _COMPLETE = {"action": "complete", "reasoning": "Done", "findings": {"summary": "ok"}} + + def test_ui_callback_never_replays_a_stale_entry(self, tmp_path: Path): + """A rejected iteration produces a 'discarded' card for that iteration, + never a re-emit of the previous entry, and the run ends without a + final duplicate of the last card.""" + calls = [] + session = self._run( + tmp_path, + [_make_llm_response(None, raw_text="not JSON"), # iter 1: rejected + _make_llm_response({"action": "dance", "reasoning": "x"}), # iter 2: rejected + _make_llm_response(self._COMPLETE)], # iter 3 + on_iteration=lambda it, entry, kind, opf: calls.append((it, kind, entry.mode)), + ) + assert [c[0] for c in calls] == [0, 1, 2, 3] + assert [c[1] for c in calls[1:3]] == ["discarded", "discarded"] + assert all(c[2] == "discarded" for c in calls[1:3]) + assert session.termination_reason == "completed" + # the synthetic cards are UI-only: the journal holds base case + complete + assert len(session.journal) == 2 + + def test_stall_breaker_is_off_by_default(self, tmp_path: Path, monkeypatch): + monkeypatch.delenv("AGENTIGRID_STALL_LIMIT", raising=False) + bad = {"action": "dance", "reasoning": "x"} + session = self._run(tmp_path, [_make_llm_response(bad)] * 3 + [_make_llm_response(self._COMPLETE)]) + assert session.termination_reason == "completed" + assert len(session.journal.discarded_actions) == 3 + + def test_stall_breaker_stops_after_n_no_progress_iterations(self, tmp_path: Path, monkeypatch): + monkeypatch.setenv("AGENTIGRID_STALL_LIMIT", "2") + bad = {"action": "dance", "reasoning": "x"} + session = self._run(tmp_path, [_make_llm_response(bad)] * 3 + [_make_llm_response(self._COMPLETE)]) + assert session.termination_reason == "stalled_no_progress" + assert [d["iteration"] for d in session.journal.discarded_actions] == [1, 2] + + def test_internal_error_in_an_action_is_discarded_and_the_run_continues(self, tmp_path: Path): + with patch.object(AgentLoopController, "_handle_analyze", side_effect=RuntimeError("boom")): + session = self._run( + tmp_path, + [_make_llm_response({"action": "analyze", "query": "max loading", "reasoning": "x"}), + _make_llm_response(self._COMPLETE)], + ) + assert session.termination_reason == "completed" + d = session.journal.discarded_actions[0] + assert (d["iteration"], d["kind"]) == (1, "internal") + assert "RuntimeError: boom" in d["feedback"] + def test_unknown_action(self, tmp_path: Path): """LLM returns unknown action, then completes.""" cfg = _make_config(tmp_path, max_iterations=10) diff --git a/agentigrid/tests/e2e/test_e2e.py b/agentigrid/tests/e2e/test_e2e.py index 5766d4e5..c09cd796 100644 --- a/agentigrid/tests/e2e/test_e2e.py +++ b/agentigrid/tests/e2e/test_e2e.py @@ -113,7 +113,7 @@ def _make_config( data=DataConfig(data_dir=DATA_DIR), llm=LLMConfig( backend="anthropic" if _has_anthropic_key else "openai", - model="claude-sonnet-4-20250514" if _has_anthropic_key else "gpt-4o-mini", + model="claude-sonnet-4-6" if _has_anthropic_key else "gpt-4o-mini", api_key_env="ANTHROPIC_API_KEY" if _has_anthropic_key else "OPENAI_API_KEY", openai_base_url=None, ollama_host="http://localhost:11434", diff --git a/agentigrid/tests/journal/test_iteration_count.py b/agentigrid/tests/journal/test_iteration_count.py new file mode 100644 index 00000000..2e3c6d54 --- /dev/null +++ b/agentigrid/tests/journal/test_iteration_count.py @@ -0,0 +1,51 @@ +"""Iteration count shown to users: agent iterations plus the base case.""" + +from __future__ import annotations + +from agentigrid.engine.journal import SearchJournal, format_iteration_count + +from tests.journal.test_journal import _make_entry + + +def _journal(iterations, discarded=()): + j = SearchJournal() + for i in iterations: + j.add_entry(_make_entry(i)) + j.discarded_actions = [{"iteration": i, "kind": "rejected"} for i in discarded] + return j + + +def test_base_case_is_not_counted_as_an_agent_iteration(): + stats = _journal([0, 1, 2]).summary_stats() + assert stats["total_iterations"] == 3 # unchanged: journal entries + assert stats["llm_iterations"] == 2 + assert stats["has_base_case"] is True + assert format_iteration_count(stats) == "2 + base case" + + +def test_rejected_iterations_still_count(): + # max 4: iteration 3 was rejected (no journal entry, kept in discarded_actions) + stats = _journal([0, 1, 2, 4], discarded=[3]).summary_stats() + assert stats["total_iterations"] == 4 + assert stats["llm_iterations"] == 4 + assert format_iteration_count(stats) == "4 + base case" + + +def test_several_entries_for_one_iteration_count_once(): + stats = _journal([0, 1, 1, 2]).summary_stats() + assert stats["llm_iterations"] == 2 + + +def test_no_base_case(): + stats = _journal([1, 2]).summary_stats() + assert format_iteration_count(stats) == "2" + + +def test_old_stats_without_new_keys_fall_back(): + assert format_iteration_count({"total_iterations": 5}) == "5" + + +def test_empty_journal(): + stats = SearchJournal().summary_stats() + assert stats["llm_iterations"] == 0 and stats["has_base_case"] is False + assert format_iteration_count(stats) == "0" diff --git a/agentigrid/tests/llm/test_backends.py b/agentigrid/tests/llm/test_backends.py index 2111ecfe..97ed40b4 100644 --- a/agentigrid/tests/llm/test_backends.py +++ b/agentigrid/tests/llm/test_backends.py @@ -233,7 +233,7 @@ class TestAnthropicLive: def test_simple_prompt(self): cfg = _dummy_config( backend="anthropic", - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-6", api_key_env="ANTHROPIC_API_KEY", ) backend = create_backend(cfg) diff --git a/agentigrid/tests/llm/test_prompt_cache.py b/agentigrid/tests/llm/test_prompt_cache.py new file mode 100644 index 00000000..0793dda8 --- /dev/null +++ b/agentigrid/tests/llm/test_prompt_cache.py @@ -0,0 +1,229 @@ +"""Prompt caching (Anthropic) and per-run token metering. + +Caching must be billing-only: the model sees the same system text either way, +``prompt_tokens`` keeps meaning TOTAL input tokens, and the cache breakdown is +reported separately. The UsageMeter must count every call without altering it. +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from agentigrid.backends import LLMBackend, LLMResponse, UsageMeter +from agentigrid.config import LLMConfig, load_config + +try: + import anthropic as _anthropic # noqa: F401 + _has_anthropic = True +except ImportError: + _has_anthropic = False + + +def _cfg(**over) -> LLMConfig: + base = dict( + backend="anthropic", model="claude-sonnet-4-6", api_key_env="TEST_API_KEY", + openai_base_url=None, ollama_host="http://localhost:11434", + ollama_cloud_host=None, temperature=0.3, max_tokens=256, + ) + base.update(over) + return LLMConfig(**base) + + +def _fake_message(input_tokens, creation, read, output_tokens=7, text='{"action":"complete"}'): + usage = SimpleNamespace( + input_tokens=input_tokens, + cache_creation_input_tokens=creation, + cache_read_input_tokens=read, + output_tokens=output_tokens, + ) + return SimpleNamespace(content=[SimpleNamespace(type="text", text=text)], usage=usage) + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +def test_prompt_cache_defaults_on(): + cfg = load_config(None) + assert cfg.llm.prompt_cache is True + + +def test_prompt_cache_can_be_disabled_by_override(): + cfg = load_config(None, cli_overrides={"llm.prompt_cache": False}) + assert cfg.llm.prompt_cache is False + + +# --------------------------------------------------------------------------- +# Anthropic backend request shape + token accounting +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_system_prompt_sent_as_cached_block(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=True)) + with patch.object(backend._client.messages, "create", + return_value=_fake_message(50, 4000, 0)) as create: + backend.complete("SYSTEM TEXT", "user text") + system = create.call_args.kwargs["system"] + assert system == [{"type": "text", "text": "SYSTEM TEXT", + "cache_control": {"type": "ephemeral"}}] + # the user turn is untouched + assert create.call_args.kwargs["messages"] == [{"role": "user", "content": "user text"}] + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_cache_off_sends_plain_string(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=False)) + with patch.object(backend._client.messages, "create", + return_value=_fake_message(4050, None, None)) as create: + resp = backend.complete("SYSTEM TEXT", "user text") + assert create.call_args.kwargs["system"] == "SYSTEM TEXT" + assert resp.prompt_tokens == 4050 + assert resp.cache_creation_tokens is None and resp.cache_read_tokens is None + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_prompt_tokens_is_total_input_with_breakdown(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=True)) + # first call writes the cache, second reads it + with patch.object(backend._client.messages, "create", + side_effect=[_fake_message(50, 4000, 0), _fake_message(60, 0, 4000)]): + first = backend.complete("S", "u1") + second = backend.complete("S", "u2") + assert (first.prompt_tokens, first.cache_creation_tokens, first.cache_read_tokens) == (4050, 4000, 0) + assert (second.prompt_tokens, second.cache_creation_tokens, second.cache_read_tokens) == (4060, 0, 4000) + assert first.json_data == {"action": "complete"} + + +# --------------------------------------------------------------------------- +# UsageMeter +# --------------------------------------------------------------------------- + +class _Stub(LLMBackend): + def __init__(self, responses): + self._responses = list(responses) + self.seen = [] + + def complete(self, system_prompt, user_prompt, temperature=None): + self.seen.append((system_prompt, user_prompt, temperature)) + return self._responses.pop(0) + + def name(self): + return "stub" + + def supports_json_mode(self): + return False + + def custom(self): + return "delegated" + + +def _resp(pt, ct, cw=None, cr=None): + return LLMResponse(raw_text="x", json_data=None, json_error=None, model="m", + backend="stub", prompt_tokens=pt, completion_tokens=ct, + cache_creation_tokens=cw, cache_read_tokens=cr) + + +def test_usage_meter_counts_all_calls_and_is_transparent(): + inner = _Stub([_resp(4050, 10, 4000, 0), _resp(4060, 12, 0, 4000), _resp(None, None)]) + meter = UsageMeter(inner) + r1 = meter.complete("S", "u1") + meter.complete("S", "u2", temperature=0.0) + meter.complete("S", "u3") + assert r1 is not None and r1.prompt_tokens == 4050 + assert inner.seen[1] == ("S", "u2", 0.0) # args passed through verbatim + assert meter.totals() == { + "calls": 3, "prompt_tokens": 8110, "completion_tokens": 22, + "cache_creation_tokens": 4000, "cache_read_tokens": 4000, + "api_errors": 0, "truncated": 0, "temperature_sent": None, + } + assert meter.name() == "stub" + assert meter.custom() == "delegated" # backend-specific attrs delegate + + +def test_journal_exports_llm_usage(tmp_path): + import json + from agentigrid.engine.journal import SearchJournal + + j = SearchJournal() + j.llm_usage = {"calls": 2, "prompt_tokens": 100, "completion_tokens": 5, + "cache_creation_tokens": 80, "cache_read_tokens": 0} + out = tmp_path / "j.json" + j.export_json(out) + assert json.loads(out.read_text())["llm_usage"]["cache_creation_tokens"] == 80 + + +def test_temperature_rejected_retries_without_it_and_remembers(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg(model="claude-sonnet-5")) + calls = [] + + def create(**kw): + calls.append(kw) + if "temperature" in kw: + raise RuntimeError("Error code: 400 - `temperature` is deprecated for this model.") + return _fake_message(10, 0, 0) + + with patch.object(backend._client.messages, "create", side_effect=create): + r1 = backend.complete("sys", "u1") + r2 = backend.complete("sys", "u2") + assert not r1.api_error and r1.json_data == {"action": "complete"} + assert ["temperature" in c for c in calls] == [True, False, False] # retried once, then omitted + assert backend.temperature_sent is False and r2.prompt_tokens == 10 + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_other_api_errors_are_flagged_and_counted(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg()) + meter = UsageMeter(backend) + with patch.object(backend._client.messages, "create", side_effect=RuntimeError("401 invalid x-api-key")): + r = meter.complete("sys", "u") + assert r.api_error and r.json_data is None + t = meter.totals() + assert t["calls"] == 1 and t["api_errors"] == 1 and t["prompt_tokens"] == 0 + with patch.object(backend._client.messages, "create", return_value=_fake_message(5, 0, 0)): + meter.complete("sys", "u") + assert meter.totals()["temperature_sent"] is True + + +# --------------------------------------------------------------------------- +# Output-token cap: stop_reason telemetry and AGENTIGRID_MAX_TOKENS +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_max_tokens_stop_is_reported_and_counted(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg()) + meter = UsageMeter(backend) + msg = _fake_message(10, 0, 0, output_tokens=256, text="") + msg.content = [SimpleNamespace(type="thinking", thinking="...")] + msg.stop_reason = "max_tokens" + with patch.object(backend._client.messages, "create", return_value=msg): + r = meter.complete("sys", "u") + assert r.stop_reason == "max_tokens" and r.content_types == ["thinking"] + assert r.raw_text == "" and r.json_data is None and not r.api_error + assert meter.totals()["truncated"] == 1 + + +def test_max_tokens_env_override(monkeypatch): + monkeypatch.setenv("AGENTIGRID_MAX_TOKENS", "16000") + assert load_config(None).llm.max_tokens == 16000 + monkeypatch.setenv("AGENTIGRID_MAX_TOKENS", "lots") + assert load_config(None).llm.max_tokens == 4096 # ignored, default kept + monkeypatch.delenv("AGENTIGRID_MAX_TOKENS") + assert load_config(None).llm.max_tokens == 4096