diff --git a/agentigrid/README.md b/agentigrid/README.md index a731bec0..ced35f1a 100644 --- a/agentigrid/README.md +++ b/agentigrid/README.md @@ -307,7 +307,7 @@ agentigrid ./data/exago/examples/case_ACTIVSg200.m "Find the maximum load scalin # With options agentigrid ./data/exago/examples/case_ACTIVSg200.m "Minimize generation cost" \ - --backend anthropic --model claude-sonnet-4-20250514 \ + --backend anthropic --model claude-sonnet-4-6 \ --app opflow --max-iter 30 --verbose # Quiet mode (only show final summary) @@ -364,7 +364,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r Version 0.1.0 ============================================================ Backend: anthropic - Model: claude-sonnet-4-20250514 + Model: claude-sonnet-4-6 Application: opflow Base case: data/exago/examples/case_ACTIVSg200.m Goal: Find the maximum load scaling factor @@ -375,7 +375,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r [Iter 0] Running base case simulation... [Iter 0] Base case: CONVERGED, cost=$27,557.57 -[Iter 1] Sending prompt to anthropic (claude-sonnet-4-20250514)... +[Iter 1] Sending prompt to anthropic (claude-sonnet-4-6)... [Iter 1] LLM action: modify — "Scale all loads +20%" [Iter 1] Applied 1 command(s), 0 skipped [Iter 1] Simulation completed in 0.04s — CONVERGED, cost=$33,019.55 @@ -390,7 +390,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r ============================================================ Goal: Find the maximum load scaling factor Application: opflow - Backend: anthropic (claude-sonnet-4-20250514) + Backend: anthropic (claude-sonnet-4-6) Iterations: 6 (of max 20) Duration: 18.3 seconds Tokens used: ~12,450 (prompt: 9,200, completion: 3,250) diff --git a/agentigrid/agentigrid/backends/__init__.py b/agentigrid/agentigrid/backends/__init__.py index dd3651b0..15114dfa 100644 --- a/agentigrid/agentigrid/backends/__init__.py +++ b/agentigrid/agentigrid/backends/__init__.py @@ -2,10 +2,10 @@ from __future__ import annotations -from agentigrid.backends.base import LLMBackend, LLMResponse +from agentigrid.backends.base import LLMBackend, LLMResponse, UsageMeter from agentigrid.config import LLMConfig -__all__ = ["LLMBackend", "LLMResponse", "create_backend"] +__all__ = ["LLMBackend", "LLMResponse", "UsageMeter", "create_backend"] def create_backend(config: LLMConfig) -> LLMBackend: diff --git a/agentigrid/agentigrid/backends/anthropic_backend.py b/agentigrid/agentigrid/backends/anthropic_backend.py index 10bbea96..594ba46d 100644 --- a/agentigrid/agentigrid/backends/anthropic_backend.py +++ b/agentigrid/agentigrid/backends/anthropic_backend.py @@ -27,6 +27,9 @@ def __init__(self, config: LLMConfig) -> None: raise ImportError("anthropic package is required for the Anthropic backend") self._config = config + # None = not yet known; True/False once a call has succeeded / the + # model has rejected the parameter (see complete()). + self.temperature_sent: Optional[bool] = None api_key = os.environ.get(config.api_key_env) if not api_key: logger.warning( @@ -40,6 +43,26 @@ def name(self) -> str: """Return the backend name.""" return "anthropic" + def _system_param(self, system_prompt: str): + """System prompt in the form the Messages API expects. + + With ``prompt_cache`` on, the system prompt is sent as a single text block + carrying an ephemeral ``cache_control`` breakpoint. AgentiGrid builds the + system prompt once per session, so every later iteration reads it from the + cache instead of paying the full input price again. This affects billing + only; the model sees identical text either way. Prompts below the model's + minimum cacheable length are silently processed uncached by the API. + """ + if not getattr(self._config, "prompt_cache", False): + return system_prompt + return [ + { + "type": "text", + "text": system_prompt, + "cache_control": {"type": "ephemeral"}, + } + ] + def supports_json_mode(self) -> bool: """Anthropic does not have a native JSON output mode.""" return False @@ -54,20 +77,45 @@ def complete( temp = temperature if temperature is not None else self._config.temperature try: - response = self._client.messages.create( + kwargs = dict( model=self._config.model, max_tokens=self._config.max_tokens, - temperature=temp, - system=system_prompt, + system=self._system_param(system_prompt), messages=[{"role": "user", "content": user_prompt}], ) + if self.temperature_sent is not False: + kwargs["temperature"] = temp + try: + response = self._client.messages.create(**kwargs) + if "temperature" in kwargs: + self.temperature_sent = True + except Exception as exc: + # Newer models reject the sampling-temperature parameter + # ("`temperature` is deprecated for this model"). Retry once + # without it and stop sending it for the rest of the session; + # the journal records temperature_sent=False. + if "temperature" in kwargs and "temperature" in str(exc).lower(): + logger.warning("Model %s rejects `temperature`; retrying without it.", self._config.model) + kwargs.pop("temperature") + self.temperature_sent = False + response = self._client.messages.create(**kwargs) + else: + raise raw_text = "" + content_types = [] for block in response.content: + content_types.append(getattr(block, "type", None)) if block.type == "text": raw_text += block.text + stop_reason = getattr(response, "stop_reason", None) + if stop_reason == "max_tokens": + logger.warning("Response stopped at max_tokens=%s (blocks: %s, %d text chars)", + self._config.max_tokens, content_types, len(raw_text)) - prompt_tokens = getattr(response.usage, "input_tokens", None) + prompt_tokens, cache_creation, cache_read = _input_token_breakdown( + response.usage + ) completion_tokens = getattr(response.usage, "output_tokens", None) json_data, json_error = extract_json(raw_text) @@ -80,6 +128,10 @@ def complete( backend=self.name(), prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, + stop_reason=stop_reason, + content_types=content_types, ) except Exception as exc: @@ -93,4 +145,23 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) + + +def _input_token_breakdown(usage) -> tuple[Optional[int], Optional[int], Optional[int]]: + """Return (total_input, cache_creation, cache_read) from an API usage object. + + With caching, the API's ``input_tokens`` counts only tokens after the last + cache breakpoint. The total input is ``input_tokens + cache_creation_input_tokens + + cache_read_input_tokens``; returning that total keeps ``prompt_tokens`` + comparable with runs made before caching was enabled. + """ + base = getattr(usage, "input_tokens", None) + creation = getattr(usage, "cache_creation_input_tokens", None) + read = getattr(usage, "cache_read_input_tokens", None) + creation = creation if isinstance(creation, int) else None + read = read if isinstance(read, int) else None + if not isinstance(base, int): + return None, creation, read + return base + (creation or 0) + (read or 0), creation, read diff --git a/agentigrid/agentigrid/backends/base.py b/agentigrid/agentigrid/backends/base.py index 32c4581a..2e681924 100644 --- a/agentigrid/agentigrid/backends/base.py +++ b/agentigrid/agentigrid/backends/base.py @@ -18,6 +18,20 @@ class LLMResponse: backend: str prompt_tokens: Optional[int] completion_tokens: Optional[int] + # Prompt-cache accounting (Anthropic). ``prompt_tokens`` always reports the + # TOTAL input tokens (uncached + cache writes + cache reads), so its meaning + # is the same with or without caching; these break that total down. + cache_creation_tokens: Optional[int] = None + cache_read_tokens: Optional[int] = None + # True when the request itself failed (network, auth, invalid request...), + # as opposed to the model answering with unparseable text. Lets the agent + # loop record API failures separately from rejected proposals. + api_error: bool = False + # Why generation stopped ("end_turn", "max_tokens", ...) and the content + # block types returned, when the backend reports them. A "max_tokens" stop + # is an output-cap limit, not a model proposal, and is recorded as such. + stop_reason: Optional[str] = None + content_types: Optional[list] = None class LLMBackend(ABC): @@ -51,3 +65,61 @@ def name(self) -> str: def supports_json_mode(self) -> bool: """Whether this backend supports a native JSON output mode.""" ... + + +class UsageMeter(LLMBackend): + """Transparent wrapper that meters token usage across EVERY backend call. + + The agent loop's own counters cover only the main per-iteration call; the + post-search analysis calls are not counted there. Wrapping the backend once + gives complete per-run totals (including prompt-cache writes and reads) for + cost accounting, without touching any call site. Behaviour is unchanged: + every call is delegated verbatim. + """ + + def __init__(self, inner: LLMBackend) -> None: + self._inner = inner + self.calls = 0 + self.prompt_tokens = 0 + self.completion_tokens = 0 + self.cache_creation_tokens = 0 + self.cache_read_tokens = 0 + self.api_errors = 0 + self.truncated = 0 + + def complete(self, system_prompt, user_prompt, temperature=None): + resp = self._inner.complete(system_prompt, user_prompt, temperature) + self.calls += 1 + if getattr(resp, "api_error", False): + self.api_errors += 1 + if getattr(resp, "stop_reason", None) == "max_tokens": + self.truncated += 1 + self.prompt_tokens += resp.prompt_tokens or 0 + self.completion_tokens += resp.completion_tokens or 0 + self.cache_creation_tokens += resp.cache_creation_tokens or 0 + self.cache_read_tokens += resp.cache_read_tokens or 0 + return resp + + def name(self) -> str: + return self._inner.name() + + def supports_json_mode(self) -> bool: + return self._inner.supports_json_mode() + + def __getattr__(self, item): # delegate anything backend-specific + if item == "_inner": # not yet set (e.g. during copy/unpickle) + raise AttributeError(item) + return getattr(self._inner, item) + + def totals(self) -> dict: + """Per-run usage totals, as written to the journal (``llm_usage``).""" + return { + "calls": self.calls, + "prompt_tokens": self.prompt_tokens, + "completion_tokens": self.completion_tokens, + "cache_creation_tokens": self.cache_creation_tokens, + "cache_read_tokens": self.cache_read_tokens, + "api_errors": self.api_errors, + "truncated": self.truncated, + "temperature_sent": getattr(self._inner, "temperature_sent", None), + } diff --git a/agentigrid/agentigrid/backends/ollama_backend.py b/agentigrid/agentigrid/backends/ollama_backend.py index f7fb1541..ae9faf6b 100644 --- a/agentigrid/agentigrid/backends/ollama_backend.py +++ b/agentigrid/agentigrid/backends/ollama_backend.py @@ -120,4 +120,5 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) diff --git a/agentigrid/agentigrid/backends/openai_backend.py b/agentigrid/agentigrid/backends/openai_backend.py index 0b4261ab..47ecb8f0 100644 --- a/agentigrid/agentigrid/backends/openai_backend.py +++ b/agentigrid/agentigrid/backends/openai_backend.py @@ -104,4 +104,5 @@ def complete( backend=self.name(), prompt_tokens=None, completion_tokens=None, + api_error=True, ) diff --git a/agentigrid/agentigrid/config.py b/agentigrid/agentigrid/config.py index bca2bade..9eff56ea 100644 --- a/agentigrid/agentigrid/config.py +++ b/agentigrid/agentigrid/config.py @@ -35,13 +35,14 @@ }, "llm": { "backend": "anthropic", - "model": "claude-sonnet-4-20250514", + "model": "claude-sonnet-4-6", "api_key_env": "ANTHROPIC_API_KEY", "openai_base_url": None, "ollama_host": "http://localhost:11434", "ollama_cloud_host": None, "temperature": 0.3, "max_tokens": 4096, + "prompt_cache": True, }, "search": { "max_iterations": 20, @@ -132,6 +133,9 @@ class LLMConfig: ollama_cloud_host: Optional[str] temperature: float max_tokens: int + # Anthropic prompt caching of the (per-session constant) system prompt. + # Billing-only: does not change model outputs. Ignored by other backends. + prompt_cache: bool = True @dataclass(frozen=True) @@ -311,6 +315,15 @@ def load_config( target[parts[-1]] = value logger.debug("Applied CLI overrides: %s", cli_overrides) + # AGENTIGRID_MAX_TOKENS overrides the output-token cap (e.g. to fix it for a + # batch of runs); the value used is recorded in the journal's llm_usage. + _mt = os.environ.get("AGENTIGRID_MAX_TOKENS", "").strip() + if _mt: + try: + merged["llm"] = {**merged["llm"], "max_tokens": int(_mt)} + except ValueError: + logger.warning("Ignoring non-integer AGENTIGRID_MAX_TOKENS=%r", _mt) + # Build frozen dataclass sections exago_path_fields = { "binary_dir", "opflow_binary", "scopflow_binary", "tcopflow_binary", diff --git a/agentigrid/agentigrid/engine/agent_loop.py b/agentigrid/agentigrid/engine/agent_loop.py index 23dae310..37b744a8 100644 --- a/agentigrid/agentigrid/engine/agent_loop.py +++ b/agentigrid/agentigrid/engine/agent_loop.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Callable, Optional -from agentigrid.backends import create_backend +from agentigrid.backends import UsageMeter, create_backend from agentigrid.backends.base import LLMBackend, LLMResponse from agentigrid.config import AppConfig from agentigrid.engine.commands import parse_command @@ -30,7 +30,7 @@ compute_pareto_labels, format_variant_results, ) -from agentigrid.engine.journal import JournalEntry, ObjectiveEntry, SearchJournal +from agentigrid.engine.journal import JournalEntry, ObjectiveEntry, SearchJournal, format_iteration_count from agentigrid.engine.metric_extractor import available_metrics, available_metrics_for_app, extract_all_metrics from agentigrid.engine.modifier import apply_modifications, build_index_maps from agentigrid.engine import sweep_metrics @@ -66,6 +66,9 @@ def _count_scenario_rows(scenario_path: Path) -> int: logger = logging.getLogger("agentigrid.engine.agent_loop") +# Cap on the raw model response kept for a discarded iteration (journal telemetry). +DISCARDED_RESPONSE_MAX = 4000 + _MAX_CONSECUTIVE_PARSE_FAILURES = 3 @@ -765,7 +768,9 @@ def __init__( on_explore: Callable[[int, list[dict]], None] | None = None, ) -> None: self._config = config - self._backend: LLMBackend = create_backend(config.llm) + # Metered so per-run token totals (incl. prompt-cache reads/writes) + # cover every LLM call, not only the main per-iteration call. + self._backend: LLMBackend = UsageMeter(create_backend(config.llm)) self._executor = SimulationExecutor(config.exago, config.output) self._journal = SearchJournal() self._quiet = quiet @@ -1217,17 +1222,67 @@ def run(self, base_case: Path, goal: str) -> SearchSession: # 3. Agent loop max_iter = self._config.search.max_iterations + # Optional stall breaker: stop after N consecutive no-progress iterations + # (iterations that record no new journal entry — e.g. the model repeating + # an invalid action). 0/unset disables it (the default), so batch runs always + # reach max_iter and stay comparable; the UI can set it. + try: + stall_limit = int(os.environ.get("AGENTIGRID_STALL_LIMIT", "0") or "0") + except ValueError: + stall_limit = 0 + stall = 0 for iteration in range(1, max_iter + 1): if self._stop_requested: session.termination_reason = "user_stopped" self._print("\nSearch stopped by user.") break - action_type, should_continue = self._iteration(iteration, goal) - # Notify callback after each iteration + self._iteration_api_error = False + self._iteration_response_text = None + self._iteration_stop_reason = None + self._iteration_content_types = None + try: + action_type, should_continue = self._iteration(iteration, goal) + except Exception as e: + import traceback + self._print( + f"\n[Iter {iteration}] Action failed with an internal error " + f"({type(e).__name__}: {e}); discarding and continuing." + ) + traceback.print_exc() + self._error_feedback = ( + f"The previous action raised an internal error and was discarded: " + f"{type(e).__name__}: {e}. Respond with a well-formed action that " + f"strictly follows the schema." + ) + action_type, should_continue = "error", True + internal_error = True + else: + internal_error = False + # Did THIS iteration record a journal entry? A discarded/invalid action + # (common with weak local models) appends nothing. + latest_entry = self._journal.latest + recorded = latest_entry is not None and latest_entry.iteration == iteration + if action_type == "error" and not recorded: + self._record_discarded(iteration, internal_error) + # Notify the UI only on a real new entry; otherwise show a transient + # discarded card (avoids re-emitting the previous entry as a phantom). if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: + if recorded: self._on_iteration(iteration, latest_entry, action_type, self._latest_opflow) + elif action_type == "error": + self._emit_discarded(iteration) + # Stall breaker: count consecutive no-progress iterations. + if recorded: + stall = 0 + elif stall_limit: + stall += 1 + if stall >= stall_limit: + session.termination_reason = "stalled_no_progress" + self._print( + f"\nStopping early: {stall} consecutive no-progress " + f"iterations (stall limit {stall_limit})." + ) + break if not should_continue: if not session.termination_reason: session.termination_reason = "completed" @@ -1239,16 +1294,12 @@ def run(self, base_case: Path, goal: str) -> SearchSession: if not session.termination_reason: session.termination_reason = "completed" - # Final notification with termination reason - if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: - self._on_iteration( - latest_entry.iteration, - latest_entry, - session.termination_reason, - self._latest_opflow, - ) + # No final on_iteration re-emit: the last real entry was already sent by + # the per-iteration callback above. Re-emitting it duplicated the last + # timeline card, and when the final iterations were discarded it made a + # stale, lower-numbered entry appear AFTER the final iteration (e.g. a + # phantom "Iteration 6" after 20). The UI reads session.termination_reason + # when the search finishes. session.end_time = datetime.now().isoformat() session.total_prompt_tokens = self._total_prompt_tokens @@ -1266,6 +1317,88 @@ def run(self, base_case: Path, goal: str) -> SearchSession: self._finalize(session, elapsed) return session + # ------------------------------------------------------------------ + # Discarded-iteration diagnostic + # ------------------------------------------------------------------ + + def _record_discarded(self, iteration: int, internal: bool = False) -> None: + """Journal telemetry for an iteration whose action produced no entry. + + Persisted: iteration, kind (rejected / internal / api_error / + truncated), stop_reason and content block types when known, the + truncated feedback sent back to the model and, when the model answered, + its raw response (the rejected proposal itself, capped at + DISCARDED_RESPONSE_MAX characters). The entry list and all summary + statistics are unaffected.""" + stop = getattr(self, "_iteration_stop_reason", None) + if internal: + kind = "internal" + elif getattr(self, "_iteration_api_error", False): + kind = "api_error" + elif stop == "max_tokens": + kind = "truncated" # output cap hit: a harness limit, not a rejected proposal + else: + kind = "rejected" + record = { + "iteration": iteration, + "kind": kind, + "feedback": (self._error_feedback or "")[:300], + } + if stop is not None: + record["stop_reason"] = stop + ctypes = getattr(self, "_iteration_content_types", None) + if ctypes is not None: + record["content_types"] = list(ctypes) + text = getattr(self, "_iteration_response_text", None) + if text is not None and kind != "api_error": # an API error has no model response + record["response"] = text[:DISCARDED_RESPONSE_MAX] + record["response_truncated"] = len(text) > DISCARDED_RESPONSE_MAX + self._journal.discarded_actions.append(record) + + def _emit_discarded(self, iteration: int) -> None: + """Emit a UI-only 'discarded' timeline card for an iteration whose action + was rejected (invalid/malformed proposal, unknown action, parse error) + and therefore recorded no journal entry. + + The synthetic entry is deliberately NOT added to ``self._journal``: it + never affects summary stats, best-iteration selection, or the persisted + record. ``mode="discarded"`` keeps ``is_solve_iteration`` + False, and ``feasible=False`` / ``objective_value=None`` keep it out of + the session_manager's best-cost tracking. It exists purely to tell the + user *why* the iteration produced nothing, instead of showing a phantom + duplicate or a blank card. + """ + if not self._on_iteration: + return + reason = (self._error_feedback or "The model returned an invalid proposal.").strip() + # The UI label must attribute the failure to the MODEL, not the code: + # the raw feedback ("Unknown action X. Valid actions: ...") is written for + # the LLM and reads like a code assertion in the timeline. Keep only the + # first clause for the headline and prefix it clearly; the full corrective + # text still shows in the card's expander (llm_reasoning). + headline = reason.split(". ")[0].splitlines()[0].strip().rstrip(".") + if not headline: + headline = "invalid proposal" + headline = headline if len(headline) <= 120 else headline[:117] + "..." + synthetic = JournalEntry( + iteration=iteration, + description=f"LLM output rejected — {headline} (no change applied)", + commands=[], + objective_value=None, + feasible=False, + convergence_status="FAILED", + violations_count=0, + voltage_min=0.0, + voltage_max=0.0, + max_line_loading_pct=0.0, + total_gen_mw=0.0, + total_load_mw=0.0, + llm_reasoning=reason, # full detail shown in the card's expander + mode="discarded", + elapsed_seconds=0.0, + ) + self._on_iteration(iteration, synthetic, "discarded", None) + # ------------------------------------------------------------------ # Single iteration # ------------------------------------------------------------------ @@ -1322,6 +1455,11 @@ def _iteration( response = self._backend.complete(system_prompt, user_prompt) logger.debug("LLM raw response: %s", response.raw_text[:500]) + # Kept for journal telemetry if this iteration's action is discarded: + # the exact proposal the parser / validator rejected. + self._iteration_response_text = response.raw_text or "" + self._iteration_stop_reason = getattr(response, "stop_reason", None) + self._iteration_content_types = getattr(response, "content_types", None) # Track tokens pt = response.prompt_tokens or 0 @@ -1334,6 +1472,11 @@ def _iteration( f"(cumulative: ~{self._total_prompt_tokens + self._total_completion_tokens:,})" ) + # A failed request (auth, invalid request, network) is not a model + # answer: journal it as an API error, not as a rejected action. + if getattr(response, "api_error", False): + self._iteration_api_error = True + # Parse JSON from response if response.json_data is None: self._consecutive_parse_failures += 1 @@ -1343,14 +1486,23 @@ def _iteration( _MAX_CONSECUTIVE_PARSE_FAILURES, response.json_error, ) - self._print(f"[Iter {iteration}] Failed to parse LLM response as JSON") + truncated = getattr(response, "stop_reason", None) == "max_tokens" + self._print(f"[Iter {iteration}] Failed to parse LLM response as JSON" + + (f" (output hit max_tokens={self._config.llm.max_tokens})" if truncated else "")) if self._consecutive_parse_failures >= _MAX_CONSECUTIVE_PARSE_FAILURES: self._print(f"[Iter {iteration}] Too many consecutive parse failures — aborting") return "error", False - self._error_feedback = ( - "Failed to parse JSON from your response. " - "Please respond with a valid JSON object." - ) + if truncated: + self._error_feedback = ( + "Your previous response was cut off at the output-token limit before " + "a complete JSON object was produced. Respond with the JSON action " + "only, keeping the reasoning brief." + ) + else: + self._error_feedback = ( + "Failed to parse JSON from your response. " + "Please respond with a valid JSON object." + ) return "error", True self._consecutive_parse_failures = 0 @@ -1477,6 +1629,14 @@ def _handle_modify( f"{skipped_count} skipped" ) + # A modify that applied zero commands changes nothing — make the timeline + # say so instead of showing a blank "No description" card. + if applied_count == 0: + if not description or description == "No description": + description = "No commands applied (no change)" + else: + description = f"{description} — no commands applied" + if all_errors: self._error_feedback = "Command errors:\n" + "\n".join(all_errors) @@ -1592,7 +1752,7 @@ def _handle_modify( self._active_steering_directives[-1]["directive"] if self._active_steering_directives else None ) - self._journal.add_from_results( + _modify_entry = self._journal.add_from_results( iteration=iteration, description=description, commands=raw_commands, @@ -1606,6 +1766,7 @@ def _handle_modify( gencost=modified_net.gencost if self._config.search.application == "pflow" else None, exago_command=_single_call_record(sim_result), ) + _modify_entry.skipped_commands = list(all_errors) # Extract tracked metrics for multi-objective tracking if opflow is not None: @@ -2043,6 +2204,11 @@ def _resolve_candidate_buses(self, spec: dict) -> tuple[Optional[list[int]], str Returns (candidates, "") on success or (None, error_message) on failure. Shared by the feasibility sweep and the boundary sweep. """ + if not isinstance(spec, dict): + return None, ( + f"candidate_set must be an object with a 'type' field, " + f"got {type(spec).__name__}: {spec!r}" + ) ctype = spec.get("type", "") if ctype == "all_buses": candidates = [b.bus_i for b in self._base_network.buses] @@ -2200,6 +2366,8 @@ def _handle_sweep( candidate_set_spec = data.get("candidate_set", {}) mutation_template = data.get("mutation", {}) feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} if not mutation_template or "action" not in mutation_template: self._error_feedback = "sweep requires a 'mutation' dict with an 'action' key." @@ -2624,6 +2792,8 @@ def _handle_boundary_sweep( pf_spec = data.get("power_factor", s.boundary_power_factor_default) candidate_set_spec = data.get("candidate_set", {}) feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} vmin = feasibility_spec.get("Vmin", 0.9) vmax = feasibility_spec.get("Vmax", 1.1) q_frac = s.boundary_gen_q_frac @@ -3228,6 +3398,8 @@ def _handle_reserve_screen( description = data.get("description", "Hot reserve / N-1 generator security") reasoning = data.get("reasoning", "") feasibility_spec = data.get("feasibility", {}) + if not isinstance(feasibility_spec, dict): + feasibility_spec = {} vmin = feasibility_spec.get("Vmin", 0.9) vmax = feasibility_spec.get("Vmax", 1.1) minimize = bool(data.get("minimize", False)) @@ -4641,6 +4813,16 @@ def _assemble_prompt( def _finalize(self, session: SearchSession, elapsed_seconds: float) -> None: """Print summary and save journal.""" + _totals = getattr(self._backend, "totals", None) + if callable(_totals): + self._journal.llm_usage = { + **_totals(), + "backend": self._backend.name(), + "model": self._config.llm.model, + "max_tokens": self._config.llm.max_tokens, + "prompt_cache": bool(getattr(self._config.llm, "prompt_cache", False)), + } + total_tokens = self._total_prompt_tokens + self._total_completion_tokens # --- Post-search goal classification via LLM --- @@ -4699,7 +4881,7 @@ def _finalize(self, session: SearchSession, elapsed_seconds: float) -> None: print(f" Application: {session.application}") print(f" Backend: {self._backend.name()} ({self._config.llm.model})") print( - f" Iterations: {stats['total_iterations']} " + f" Iterations: {format_iteration_count(stats)} " f"(of max {self._config.search.max_iterations})" ) print(f" Duration: {elapsed_seconds:.1f} seconds") @@ -4968,16 +5150,61 @@ def resume_from(self, save_dir: Path) -> SearchSession: # Continue the agent loop from last_iteration + 1 max_iter = self._config.search.max_iterations + try: + stall_limit = int(os.environ.get("AGENTIGRID_STALL_LIMIT", "0") or "0") + except ValueError: + stall_limit = 0 + stall = 0 for iteration in range(last_iteration + 1, max_iter + 1): if self._stop_requested: session.termination_reason = "user_stopped" self._print("\nSearch stopped by user.") break - action_type, should_continue = self._iteration(iteration, goal) + self._iteration_api_error = False + self._iteration_response_text = None + self._iteration_stop_reason = None + self._iteration_content_types = None + try: + action_type, should_continue = self._iteration(iteration, goal) + except Exception as e: + import traceback + self._print( + f"\n[Iter {iteration}] Action failed with an internal error " + f"({type(e).__name__}: {e}); discarding and continuing." + ) + traceback.print_exc() + self._error_feedback = ( + f"The previous action raised an internal error and was discarded: " + f"{type(e).__name__}: {e}. Respond with a well-formed action that " + f"strictly follows the schema." + ) + action_type, should_continue = "error", True + internal_error = True + else: + internal_error = False + + # Only emit when this iteration recorded a new entry (see run() above). + latest_entry = self._journal.latest + recorded = latest_entry is not None and latest_entry.iteration == iteration + if action_type == "error" and not recorded: + self._record_discarded(iteration, internal_error) if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: + if recorded: self._on_iteration(iteration, latest_entry, action_type, self._latest_opflow) + elif action_type == "error": + self._emit_discarded(iteration) + # Stall breaker: count consecutive no-progress iterations. + if recorded: + stall = 0 + elif stall_limit: + stall += 1 + if stall >= stall_limit: + session.termination_reason = "stalled_no_progress" + self._print( + f"\nStopping early: {stall} consecutive no-progress " + f"iterations (stall limit {stall_limit})." + ) + break if not should_continue: if not session.termination_reason: session.termination_reason = "completed" @@ -4989,14 +5216,8 @@ def resume_from(self, save_dir: Path) -> SearchSession: if not session.termination_reason: session.termination_reason = "completed" - # Final notification - if self._on_iteration: - latest_entry = self._journal.latest - if latest_entry: - self._on_iteration( - latest_entry.iteration, latest_entry, - session.termination_reason, self._latest_opflow, - ) + # No final on_iteration re-emit (see run() above) — it only duplicated + # the last timeline card. session.end_time = datetime.now().isoformat() session.total_prompt_tokens = self._total_prompt_tokens diff --git a/agentigrid/agentigrid/engine/journal.py b/agentigrid/agentigrid/engine/journal.py index 4aa8a97a..0a248cb3 100644 --- a/agentigrid/agentigrid/engine/journal.py +++ b/agentigrid/agentigrid/engine/journal.py @@ -54,6 +54,10 @@ class JournalEntry: exago_command: Optional[dict] = None # Reproducible ExaGO invocation record (JSON journal only; see add_* methods) contingency_meta: Optional[dict] = None # C.5: target bus, neighbors, order, pass/fail counts reserve_meta: Optional[dict] = None # C.8: hot-reserve / N-1 generator security accounting + # Modify iterations: commands the LLM proposed that were NOT applied (parse + # errors or validator skips), as messages. [] = every proposed command was + # applied; None = not recorded (non-modify entries, older journals). + skipped_commands: Optional[list[str]] = None def is_solve_iteration(entry: JournalEntry) -> bool: @@ -187,6 +191,11 @@ def __init__(self) -> None: self.session_best: Optional[dict] = None # load_factor: session-level load scaling factor (updated by set_load_factor action). self.load_factor: Optional[float] = None + # discarded_actions: iterations whose LLM action recorded no entry -- + # rejected by the deterministic validator / parser ("rejected"), a failed + # API request ("api_error"), output cut at max_tokens ("truncated") or + # an internal error ("internal"). Exported with the journal. + self.discarded_actions: list[dict] = [] def update_session_best( self, @@ -1075,6 +1084,10 @@ def export_json(self, path: Path) -> None: data["session_best"] = self.session_best if self.load_factor is not None: data["load_factor"] = self.load_factor + if getattr(self, "llm_usage", None) is not None: + data["llm_usage"] = self.llm_usage + if getattr(self, "discarded_actions", None): + data["discarded_actions"] = self.discarded_actions path.write_text(json.dumps(data, indent=2), encoding="utf-8") logger.info("Journal exported to %s (%d entries)", path, len(self._entries)) @@ -1092,7 +1105,7 @@ def export_csv(self, path: Path) -> None: "mode", "elapsed_seconds", "timestamp", "steering_directive", "tracked_metrics", "feasibility_detail", "solver", "num_steps", "num_scenarios", "explored_variants", "candidate_count", "feasible_buses", "exago_command", - "contingency_meta", "reserve_meta", + "contingency_meta", "reserve_meta", "skipped_commands", ] with open(path, "w", newline="", encoding="utf-8") as f: @@ -1107,6 +1120,7 @@ def export_csv(self, path: Path) -> None: row["exago_command"] = json.dumps(row.get("exago_command") or None) row["contingency_meta"] = json.dumps(row.get("contingency_meta") or None) row["reserve_meta"] = json.dumps(row.get("reserve_meta") or None) + row["skipped_commands"] = json.dumps(row.get("skipped_commands")) writer.writerow(row) logger.info("Journal CSV exported to %s (%d entries)", path, len(self._entries)) @@ -1115,6 +1129,22 @@ def export_csv(self, path: Path) -> None: # Summary statistics # ------------------------------------------------------------------ + def _iteration_counts(self) -> tuple[int, bool]: + """Agent iterations actually run, and whether a base case was solved. + + ``total_iterations`` counts journal entries: it includes the base-case + solve (iteration 0) and leaves out iterations whose LLM output was + rejected (they are kept in ``discarded_actions``, not as entries). + This counts distinct iteration numbers >= 1 across both, which is the + number of agent iterations the user asked for and the run performed. + """ + numbers = {e.iteration for e in self._entries} + for d in getattr(self, "discarded_actions", None) or []: + it = d.get("iteration") if isinstance(d, dict) else None + if isinstance(it, int): + numbers.add(it) + return sum(1 for n in numbers if n >= 1), 0 in numbers + def summary_stats( self, best_iteration_override: int | None = None, @@ -1134,9 +1164,12 @@ def summary_stats( feasible_count, infeasible_count, objective_trend, voltage_range_trend, goal_type. """ + llm_iterations, has_base_case = self._iteration_counts() if not self._entries: return { "total_iterations": 0, + "llm_iterations": llm_iterations, + "has_base_case": has_base_case, "best_objective": None, "best_iteration": None, "best_bus": None, @@ -1194,6 +1227,8 @@ def summary_stats( return { "total_iterations": len(self._entries), + "llm_iterations": llm_iterations, + "has_base_case": has_base_case, "best_objective": best_objective, "best_iteration": best_iteration, "best_bus": best_bus, @@ -1248,3 +1283,15 @@ def _best_cost_in_sweeps(self) -> tuple[float, int, int | None] | None: if best is None or c < best[0]: best = (c, e.iteration, v.get("bus")) return best + + +def format_iteration_count(stats: dict) -> str: + """Display form of the iteration count, e.g. ``"2 + base case"``. + + Falls back to ``total_iterations`` for stats dicts written before + ``llm_iterations`` existed. + """ + if "llm_iterations" not in stats: + return str(stats.get("total_iterations", 0)) + n = stats["llm_iterations"] + return f"{n} + base case" if stats.get("has_base_case") else str(n) diff --git a/agentigrid/agentigrid/engine/regenerate_report.py b/agentigrid/agentigrid/engine/regenerate_report.py index 43cbd790..b456f024 100644 --- a/agentigrid/agentigrid/engine/regenerate_report.py +++ b/agentigrid/agentigrid/engine/regenerate_report.py @@ -110,6 +110,7 @@ def load_journal_export(path: Path) -> SearchJournal: exago_command=ed.get("exago_command"), contingency_meta=ed.get("contingency_meta"), reserve_meta=ed.get("reserve_meta"), + skipped_commands=ed.get("skipped_commands"), )) # Rebuild the objective registry, then overwrite its history with the saved diff --git a/agentigrid/agentigrid/engine/session_io.py b/agentigrid/agentigrid/engine/session_io.py index a5b3fe51..7971d4eb 100644 --- a/agentigrid/agentigrid/engine/session_io.py +++ b/agentigrid/agentigrid/engine/session_io.py @@ -184,6 +184,7 @@ def load_session(save_dir: Path) -> dict[str, Any]: exago_command=entry_data.get("exago_command"), contingency_meta=entry_data.get("contingency_meta"), reserve_meta=entry_data.get("reserve_meta"), + skipped_commands=entry_data.get("skipped_commands"), ) journal_entries.append(entry) diff --git a/agentigrid/agentigrid/parsers/matpower_parser.py b/agentigrid/agentigrid/parsers/matpower_parser.py index dc381b0b..23b470e5 100644 --- a/agentigrid/agentigrid/parsers/matpower_parser.py +++ b/agentigrid/agentigrid/parsers/matpower_parser.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import math import re from pathlib import Path @@ -22,10 +23,30 @@ _FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*(\w+)", re.MULTILINE) _VERSION_RE = re.compile(r"mpc\.version\s*=\s*'([^']+)'") -_BASEMVA_RE = re.compile(r"mpc\.baseMVA\s*=\s*([\d.]+)") +# The value is any numeric token (sign, exponent, Inf/NaN) up to ";" or a comment. +_BASEMVA_RE = re.compile(r"^[ \t]*mpc\.baseMVA[ \t]*=[ \t]*([^;%\n]*)", re.MULTILINE) # Match mpc.
= [ or mpc.
= { _SECTION_RE = re.compile(r"^mpc\.(\w+)\s*=\s*([{\[])", re.MULTILINE) +# Only MATPOWER case format version 2 is supported (ExaGO reads version 2). +_SUPPORTED_VERSIONS = ("2",) + +# Sections parsed into dataclasses; any other section is kept as raw text. +_KNOWN_SECTIONS = ("bus", "gen", "branch", "gencost") +# Column limits per known section. Bus rows have 13 input columns plus up to +# 4 optional solution columns (lam_P, lam_Q, mu_Vmax, mu_Vmin); gen and +# branch rows may carry any number of extra columns, which are kept. +_MIN_COLS = {"bus": 13, "gen": 10, "branch": 13, "gencost": 4} +_MAX_COLS = {"bus": 17} +# 0-based columns that must hold integers (MATPOWER BUS_I, BUS_TYPE, +# BUS_AREA, ZONE; GEN_BUS, GEN_STATUS; F_BUS, T_BUS, BR_STATUS; MODEL, NCOST). +_INT_COLS = { + "bus": (0, 1, 6, 10), + "gen": (0, 7), + "branch": (0, 1, 10), + "gencost": (0, 3), +} + def _parse_float(s: str) -> float: """Parse a string as float, handling MATLAB Inf/NaN.""" @@ -39,21 +60,61 @@ def _parse_float(s: str) -> float: return float(sl) -def _split_data_row(line: str) -> list[str]: - """Split a MATPOWER data row into tokens, stripping comments and semicolons.""" - # Remove inline comment - idx = line.find("%") - if idx >= 0: - line = line[:idx] - # Remove trailing semicolons and whitespace - line = line.rstrip().rstrip(";").rstrip() - return line.split() - +def _parse_rows(section: str, block: str) -> list[list[float]]: + """Parse the rows of a known numeric section (one row per line). -def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, int]: + Anything that would otherwise be misread raises ``ValueError`` naming the + section and the 1-based data row: unsupported syntax (several rows on one + line, commas, ``...`` continuations), non-numeric values, a column count + outside the section's limits, or a non-integral value in an integer column. + Rows are never skipped silently. + """ + rows: list[list[float]] = [] + for line in block.split("\n"): + idx = line.find("%") + if idx >= 0: + line = line[:idx] + line = line.strip().rstrip(";").rstrip() + if not line: + continue + where = f"mpc.{section} row {len(rows) + 1}" + if "..." in line: + raise ValueError(f"{where}: line continuation '...' is not supported") + if ";" in line: + raise ValueError( + f"{where}: several rows on one line are not supported; " + "put each row on its own line" + ) + if "," in line: + raise ValueError( + f"{where}: comma-separated values are not supported; " + "separate values with spaces or tabs" + ) + try: + values = [_parse_float(t) for t in line.split()] + except ValueError: + raise ValueError(f"{where}: non-numeric value in {line!r}") from None + lo, hi = _MIN_COLS[section], _MAX_COLS.get(section) + if len(values) < lo or (hi is not None and len(values) > hi): + expected = f"{lo}-{hi}" if hi is not None else f"at least {lo}" + raise ValueError(f"{where}: {len(values)} columns, expected {expected}") + for col in _INT_COLS[section]: + v = values[col] + if not math.isfinite(v) or v != int(v): + raise ValueError(f"{where}: column {col + 1} must be an integer, got {v!r}") + rows.append(values) + return rows + + +def _extract_section_block( + text: str, start: int, bracket: str, section: str = "?" +) -> tuple[str, int]: """Extract text from *start* until the matching closing bracket. Returns (block_content, end_position). + + Raises: + ValueError: If the block is never closed. """ close = "]" if bracket == "[" else "}" # Find the end of the section @@ -66,6 +127,8 @@ def _extract_section_block(text: str, start: int, bracket: str) -> tuple[str, in elif ch == close: depth -= 1 i += 1 + if depth > 0: + raise ValueError(f"mpc.{section}: '{bracket}' block is not terminated") return text[start:i - 1], i @@ -80,6 +143,11 @@ def _collect_section_comment(text: str, section_start: int) -> str: else: break comment_lines.reverse() + # Drop surrounding blank lines so a read/write round trip is stable. + while comment_lines and not comment_lines[0].strip(): + comment_lines.pop(0) + while comment_lines and not comment_lines[-1].strip(): + comment_lines.pop() return "\n".join(comment_lines) @@ -180,7 +248,8 @@ def parse_matpower(path: Path) -> MATNetwork: if not path.exists(): raise FileNotFoundError(f"MATPOWER file not found: {path}") - text = path.read_text(encoding="utf-8") + # Normalise line endings: CRLF (Windows-authored) files read the same as LF. + text = path.read_text(encoding="utf-8").replace("\r\n", "\n") # --- Function name --- m = _FUNC_RE.search(text) @@ -191,12 +260,21 @@ def parse_matpower(path: Path) -> MATNetwork: # --- Version --- m = _VERSION_RE.search(text) version = m.group(1) if m else "2" + if version not in _SUPPORTED_VERSIONS: + raise ValueError( + f"Unsupported MATPOWER case version '{version}' in {path}; " + f"supported: {', '.join(_SUPPORTED_VERSIONS)}" + ) # --- baseMVA --- m = _BASEMVA_RE.search(text) if not m: raise ValueError(f"Cannot find mpc.baseMVA in {path}") - baseMVA = float(m.group(1)) + token = m.group(1).strip() + try: + baseMVA = _parse_float(token) + except ValueError: + raise ValueError(f"mpc.baseMVA: invalid value {token!r} in {path}") from None # --- Header comments (everything before mpc.version or mpc.baseMVA) --- first_mpc = text.find("mpc.") @@ -222,32 +300,33 @@ def parse_matpower(path: Path) -> MATNetwork: if section_name in ("version", "baseMVA"): continue - block_content, _ = _extract_section_block(text, block_start, bracket) + block_content, _ = _extract_section_block(text, block_start, bracket, section_name) # Collect preceding comment line_start = text.rfind("\n", 0, match.start()) + 1 comment = _collect_section_comment(text, line_start) _section_comments[section_name] = comment + prefix = comment + "\n" if comment else "" if bracket == "{": # Cell array (bus_name, gentype, genfuel, etc.) → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + ("}" if bracket == "{" else "]") + # Keep the MATLAB terminator "};" -- ExaGO's reader finds the end of + # mpc.genfuel by searching for "};", so a bare "}" makes it skip the + # fuel types (and with them the default ramp rates and the + # renewable Pmin=0 relaxation). + raw_text = prefix + text[match.start():block_start] + block_content + "};" + extra_sections[section_name] = raw_text + continue + + if section_name not in _KNOWN_SECTIONS: + # Unknown numeric section (areas, dcline, ...) → store as raw + raw_text = prefix + text[match.start():block_start] + block_content + "];" extra_sections[section_name] = raw_text + logger.debug("Stored unknown section mpc.%s as raw text", section_name) continue - # Numeric matrix section — parse rows - rows: list[list[float]] = [] - for line in block_content.split("\n"): - stripped = line.strip() - if not stripped or stripped.startswith("%"): - continue - tokens = _split_data_row(line) - if not tokens: - continue - try: - rows.append([_parse_float(t) for t in tokens]) - except ValueError: - logger.warning("Skipping unparseable row in mpc.%s: %s", section_name, line.strip()) + # Known numeric matrix section — parse rows (raises on bad input) + rows = _parse_rows(section_name, block_content) if section_name == "bus": buses = [_row_to_bus(r) for r in rows] @@ -261,11 +340,9 @@ def parse_matpower(path: Path) -> MATNetwork: elif section_name == "gencost": gencost = [_row_to_gencost(r) for r in rows] logger.debug("Parsed %d gencost entries", len(gencost)) - else: - # Unknown numeric section → store as raw - raw_text = comment + "\n" + text[match.start():block_start] + block_content + "]" - extra_sections[section_name] = raw_text - logger.debug("Stored unknown section mpc.%s as raw text", section_name) + + if "bus" not in _section_comments: + raise ValueError(f"Cannot find mpc.bus in {path}") # Store section comments on the network for the writer net = MATNetwork( diff --git a/agentigrid/agentigrid/parsers/matpower_writer.py b/agentigrid/agentigrid/parsers/matpower_writer.py index 10bef308..c402ea0a 100644 --- a/agentigrid/agentigrid/parsers/matpower_writer.py +++ b/agentigrid/agentigrid/parsers/matpower_writer.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +import math +import re from pathlib import Path from agentigrid.parsers.matpower_model import ( @@ -15,12 +17,30 @@ logger = logging.getLogger("agentigrid.parsers.matpower") +_FUNC_RE = re.compile(r"^function\s+mpc\s*=\s*\w+", re.MULTILINE) + def _fmt(value: float) -> str: - """Format a numeric value, writing integers without decimals.""" - if value == int(value) and abs(value) < 1e15: - return str(int(value)) - return f"{value:.10g}" + """Format a numeric value, writing integers without decimals. + + Infinities and NaN are written as MATLAB ``Inf`` / ``-Inf`` / ``NaN`` + (the parser reads them back). + """ + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Inf" if value > 0 else "-Inf" + text = f"{value:.10g}" + rounded = float(text) + if math.isinf(rounded): + # Rounding to 10 digits pushed a value near the float maximum past + # it; that text would read back as Inf. Write it exactly instead. + text, rounded = repr(value), value + # Decide the integer form on the ROUNDED value, so a value that rounds to + # an integer is written the same way the second time (idempotence). + if rounded == int(rounded) and abs(rounded) < 1e15: + return str(int(rounded)) + return text def _write_bus_row(b: Bus) -> str: @@ -31,7 +51,7 @@ def _write_bus_row(b: Bus) -> str: ] for val in (b.lam_P, b.lam_Q, b.mu_Vmax, b.mu_Vmin): if val is not None: - parts.append(f"{val:.4f}") + parts.append(_fmt(val)) return "\t" + "\t".join(parts) + ";" @@ -64,6 +84,19 @@ def _write_gencost_row(gc: GenCost) -> str: return "\t" + "\t".join(parts) + ";" +def _terminate(raw: str) -> str: + """Ensure a raw section ends with "};" / "];". + + ExaGO locates the end of ``mpc.genfuel`` by the literal "};". Networks + parsed by older versions stored the block without the semicolon; writing + that back made ExaGO ignore the fuel types, leaving every ramp rate at 0 + (SCOPFLOW cannot redispatch in contingencies) and wind/solar fixed at Pmax. + """ + if raw.endswith("}") or raw.endswith("]"): + return raw + ";" + return raw + + def write_matpower(network: MATNetwork, path: Path) -> None: """Write a MATNetwork object to a MATPOWER .m file. @@ -80,14 +113,22 @@ def write_matpower(network: MATNetwork, path: Path) -> None: lines: list[str] = [] # --- Header --- - lines.append(network.header_comments.rstrip("\n")) - # Ensure function line is present (header_comments includes it) - if f"function mpc = {network.casename}" not in network.header_comments: - lines.append(f"function mpc = {network.casename}") + # The function line always carries network.casename: an existing one in + # header_comments is renamed, a missing one is added as the first line. + header = network.header_comments.rstrip("\n") + func_line = f"function mpc = {network.casename}" + if _FUNC_RE.search(header): + header = _FUNC_RE.sub(lambda _m: func_line, header, count=1) + else: + header = func_line + ("\n" + header if header else "") + lines.append(header) # --- Version & baseMVA --- - lines.append("") - lines.append(f"%% MATPOWER Case Format : Version {network.version}") + # The header read back from a file already holds the format comment; + # adding it again would grow the file on every read/write round trip. + if "MATPOWER Case Format" not in header: + lines.append("") + lines.append(f"%% MATPOWER Case Format : Version {network.version}") lines.append(f"mpc.version = '{network.version}';") lines.append("") lines.append("%%----- Power Flow Data -----%%") @@ -137,9 +178,10 @@ def write_matpower(network: MATNetwork, path: Path) -> None: # --- Extra sections (gentype, genfuel, bus_name, etc.) --- for section_name, raw_text in network.extra_sections.items(): lines.append("") - lines.append(raw_text.rstrip()) + lines.append(_terminate(raw_text.rstrip())) lines.append("") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") + # Always LF line endings, whatever the platform or the source file used. + path.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") logger.info("Wrote MATPOWER file: %s", path) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 5c83fa3a..f7a41952 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,94 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## MATPOWER reader/writer hardening — no silent misreads, stable round trips (2026-10-05) + +Found while writing an independent spec for the parser and writer (round-trip, error and syntax rules). Every real case in ExaGO's `datafiles/` (29 case files, `case5` to `case_ACTIVSg10k`, including CRLF ones) still parses and round-trips to an equivalent network; the files are already accepted by the stricter rules. + +| # | Defect | Fix | +|---|---|---| +| 1 | A row with a non-numeric value was skipped with only a log warning, so a bus, generator or branch could silently vanish. Too-short rows crashed with `IndexError`. | `_parse_rows()` raises `ValueError` naming the section and 1-based row for non-numeric values, column counts outside the section's limits (bus 13–17; gen ≥ 10; branch ≥ 13; gencost ≥ 4) and non-integral/Inf/NaN values in integer columns. | +| 2 | Valid MATLAB syntax was misread: two rows on one line became one garbage row; comma-separated rows were all dropped; `...` continuations crashed. | These raise `ValueError` with a clear message. An unterminated `[`/`{` block and a missing `mpc.bus` raise as well. | +| 3 | The writer crashed on `Inf`/`NaN` (`OverflowError`/`ValueError`), which the reader accepts. | Written as `Inf`, `-Inf`, `NaN`. Bus solution columns (`lam_*`, `mu_*`) use the same 10-significant-digit format instead of `.4f`. | +| 4 | Every read/write round trip added another `%% MATPOWER Case Format` line, and blank lines accumulated before raw sections. | The format comment is written only if the header lacks it; collected section comments drop surrounding blank lines. Writing is idempotent. | +| 5 | MATPOWER case version `'1'` was accepted although ExaGO reads version 2. | Raises `ValueError`; a missing version still means `'2'`. | + +Also: CRLF input is normalised on read and the writer always emits LF; the `function mpc = …` line always carries `network.casename` (renamed if the header has another name, added first if missing). Unknown numeric sections (`areas`, `dcline`, …) stay opaque raw text and are not validated. + +| File | Change | +|---|---| +| `agentigrid/parsers/matpower_parser.py` | Fixes 1, 2, 4 (comments), 5; CRLF normalisation | +| `agentigrid/parsers/matpower_writer.py` | Fixes 3, 4 (header); casename/function line; LF output | +| `tests/grid_files/test_matpower_hardening.py` | New (29 tests; 22 fail without the fix) | + +**Follow-up (property-based testing).** Randomised round-trip tests then found three more defects: + +- A finite value near the float maximum was written with 10 significant digits that round past it, and read back as `Inf`. The writer now falls back to the exact representation. +- A value that rounds to an integer (for example `9999999999.5`) was written as `1e+10` the first time and `10000000000` the second, so writing was not idempotent. The integer form is now decided on the rounded value. +- `mpc.baseMVA` was read with a digits-and-dots pattern. `1e2` (or `1e+15`, which the writer itself emits) was silently read as `1`, and negative values were "not found". It now accepts any numeric literal (sign, exponent, `Inf`/`NaN`) on an `mpc.baseMVA` line, ignores the text in comments, and raises `ValueError` for a non-numeric value. + +18 more tests; 12 of them fail without these fixes. + +--- + +## MATPOWER cell arrays keep "};" — ExaGO reads genfuel again (2026-10-03) + +**Symptom.** SCOPFLOW reported local infeasibility on AgentiGrid-rewritten ACTIVSg200 at base load, while the original file converged. The rewritten file had `mpc.genfuel = { ... }` with a bare `}`. + +**Cause.** `src/ps/psreaddata.cpp` ends `mpc.genfuel` at the first line containing `};`. The parser stored cell arrays (`gentype`, `genfuel`, `bus_name`) without the trailing `;` and the writer wrote them back as `}`, so for every case AgentiGrid rewrote, ExaGO ignored the fuel types: all units took the default fuel, ramp rates were 0, and wind/solar stayed at Pmin = Pmax. + +**Evidence in the repository.** `tests/fixtures/sample_opflow_output.txt` (OPFLOW on AgentiGrid-rewritten ACTIVSg200) listed all 49 generators as COAL; `case_ACTIVSg200.m` has 25 coal, 17 ng, 6 wind and 1 nuclear (bus 189). `test_gen_189` asserted COAL, so it encoded the bug. + +### Changes + +| File | Change | +|---|---| +| `agentigrid/parsers/matpower_parser.py` | Keep `};` / `];` on stored cell-array sections | +| `agentigrid/parsers/matpower_writer.py` | Terminate cell arrays with `};`, including raw sections from networks parsed before the fix | +| `tests/grid_files/test_matpower_cell_terminators.py` | New (3 tests): mimics ExaGO's genfuel detection on a round-tripped ACTIVSg200; all fail without the fix | +| `tests/fixtures/sample_opflow_output.txt` | Regenerated with the fixed writer (Ipopt 3.14.14): real fuel mix | +| `tests/results/test_opflow_parser.py` | Expected values for the new fixture (21 iterations, 0.021 s, bus 189 NUCLEAR); new `test_fuel_mix_matches_case_file` | +| `tests/runner/test_executor.py` | `TestExecutorLive` skips without the `opflow` binary (like `TestMPIGuard`) and no longer overwrites the tracked fixture on every run; regenerate with `AGENTIGRID_REGENERATE_FIXTURES=1` | +| `tests/README.md` | How to regenerate the fixture | + +Fast suite (`python -m pytest -m "not slow"`), without ExaGO binaries: **1083 passed, 13 skipped**. Before this change the same run had 2 failures: `test_solve_time` (the committed fixture no longer matched its expected value) and `test_opflow_run` (no skip without the binary). + +--- + +## Agent loop and LLM backends: robust iterations, honest UI, usage accounting (2026-10-03) + +**Motivation.** Long runs with local models (Ollama) showed iterations that went wrong in ways the UI hid or misreported: a malformed action crashed the run; a rejected iteration re-emitted the previous card, so the timeline showed stale or out-of-order cards (e.g. "Iteration 6" after iteration 20); analysis and completion entries were labelled FAILED; a modify that applied no command showed "No description"; and the summary counted the base case as an iteration ("Iterations 3" for max 2). Separately, Anthropic runs re-sent the same system prompt at full price on every iteration, and some newer models reject the `temperature` parameter. + +### Agent loop (`agentigrid/engine/agent_loop.py`) + +- Each iteration runs inside a guard: an exception in an action handler is discarded with feedback to the model, and the run continues. Handlers that read `feasibility` / `candidate_set` tolerate non-dict values. +- The UI callback fires only when the iteration recorded a journal entry. Otherwise a UI-only "discarded" card explains why ("LLM output rejected — … (no change applied)"); it is never added to the journal. The final re-emit of the last entry is gone (it duplicated the last card). Same in `resume`. +- `journal.discarded_actions` records every iteration that produced no entry: kind (`rejected`, `api_error`, `truncated`, `internal`), the feedback sent to the model, `stop_reason`, content block types and the raw response (capped at 4,000 characters). Entries and summary statistics are unchanged. +- A response cut at `max_tokens` gets its own message and feedback asking for the JSON action only. +- A modify that applies zero commands says so in its description; modify entries record `skipped_commands`. +- Optional stall breaker: `AGENTIGRID_STALL_LIMIT=N` ends the run after N consecutive iterations without a new entry (`termination_reason = "stalled_no_progress"`). Off by default. + +### Backends (`agentigrid/backends/`) + +- Anthropic prompt caching of the per-session system prompt (`llm.prompt_cache`, default on). Billing only; the model sees the same text. `prompt_tokens` stays the total input; cache writes and reads are reported separately. +- `UsageMeter` wraps the backend and counts every call (including post-search analysis); totals go to the journal as `llm_usage`. +- Models that reject `temperature` are retried once without it, and it is not sent again in that session. +- `LLMResponse` gains `api_error`, `stop_reason`, `content_types` and cache token fields; failed requests in all backends set `api_error`. +- `AGENTIGRID_MAX_TOKENS` overrides the output-token cap. + +### UI and reports + +- Iterations shown as "N + base case" (UI summary and live panel, CLI summary, PDF report). `summary_stats()` adds `llm_iterations` and `has_base_case`; `total_iterations` is unchanged. +- Timeline cards show FAILED only for solves that did not converge, not for analysis / complete / sweep / contingency / explore entries or discarded proposals. +- Sidebar checkbox for the stall breaker (unchecked by default). +- Default Anthropic model `claude-sonnet-4-20250514` (retired) → `claude-sonnet-4-6`. + +### Tests + +`tests/llm/test_prompt_cache.py` (new: caching, token breakdown, `UsageMeter`, temperature retry, API errors, max_tokens, env override), `tests/journal/test_iteration_count.py` (new), `tests/agent_loop/test_agent_loop.py` (+8: discarded telemetry, API errors, truncation, response cap, no stale UI cards, stall breaker off/on, internal errors). The 4 new agent-loop callback/stall/error tests fail on the previous agent loop. + +--- + ## C.2/C.3 corrections — metric gating, summary aggregation, predicate audit, sweep dedup (2026-06-23) Four contained fixes from the prompt-13/15/17 runs. No capability behavior changed (capacity numbers, dispatch results, metric/predicate definitions, and the C.1 binding identifier are untouched) — only how results are gated, summarized, audited, and deduplicated. diff --git a/agentigrid/configs/default_config.yaml.template b/agentigrid/configs/default_config.yaml.template index 9ef1edfc..d076872d 100644 --- a/agentigrid/configs/default_config.yaml.template +++ b/agentigrid/configs/default_config.yaml.template @@ -23,7 +23,7 @@ data: # LLM backend settings llm: backend: "anthropic" # openai | anthropic | ollama | ollama-cloud - model: "claude-sonnet-4-20250514" + model: "claude-sonnet-4-6" api_key_env: "ANTHROPIC_API_KEY" # env var name containing the API key # OpenAI-specific openai_base_url: null @@ -33,6 +33,7 @@ llm: # Generation parameters temperature: 0.3 max_tokens: 4096 + prompt_cache: true # Anthropic only: cache the system prompt (billing-only; outputs unchanged) # Search settings search: diff --git a/agentigrid/configs/local_config.yaml.template b/agentigrid/configs/local_config.yaml.template index 7442d60e..43566e21 100644 --- a/agentigrid/configs/local_config.yaml.template +++ b/agentigrid/configs/local_config.yaml.template @@ -22,6 +22,7 @@ llm: # ollama_cloud_host: null # set for Ollama Cloud temperature: 0.3 max_tokens: 4096 + prompt_cache: true # Anthropic only: cache the system prompt (billing-only; outputs unchanged) # Search settings search: diff --git a/agentigrid/launcher/ARCHITECTURE.md b/agentigrid/launcher/ARCHITECTURE.md index f8fa76d3..ecb9f7c9 100644 --- a/agentigrid/launcher/ARCHITECTURE.md +++ b/agentigrid/launcher/ARCHITECTURE.md @@ -224,7 +224,7 @@ Always visible. Contains all search parameters. **Section: LLM Backend** - Backend selector: `anthropic` | `openai` | `ollama` | `ollama-cloud` -- Model name text input (pre-filled with defaults per backend: `claude-sonnet-4-20250514` for Anthropic, `gpt-4o` for OpenAI, etc.) +- Model name text input (pre-filled with defaults per backend: `claude-sonnet-4-6` for Anthropic, `gpt-4o` for OpenAI, etc.) - Temperature slider (0.0 – 1.0, default 0.3) **Section: Search Parameters** @@ -274,7 +274,7 @@ Shown while a search is running. Two-column layout: - Key metrics: voltage range, max line loading, gen/load totals - Mode used (fresh/accumulative) - Current iteration shows a live status indicator: - - Spinner with phase text: "Sending prompt to Claude (claude-sonnet-4-20250514)..." → "Applying 3 modifications..." → "Running OPFLOW simulation..." → "Parsing results..." + - Spinner with phase text: "Sending prompt to Claude (claude-sonnet-4-6)..." → "Applying 3 modifications..." → "Running OPFLOW simulation..." → "Parsing results..." **Right column (~35%)** — Live Charts: - **Convergence chart** (Plotly line chart): Objective value vs. iteration number diff --git a/agentigrid/launcher/app.py b/agentigrid/launcher/app.py index 76339485..92fc4e05 100644 --- a/agentigrid/launcher/app.py +++ b/agentigrid/launcher/app.py @@ -37,6 +37,7 @@ ) from agentigrid.parsers import parse_matpower, network_summary +from agentigrid.engine.journal import NON_SOLVE_STATUSES, format_iteration_count # ── Page Configuration ─────────────────────────────────────────────────────── @@ -394,6 +395,23 @@ def render_sidebar() -> dict: max_iterations = st.slider( "Max iterations", 1, 50, 20, disabled=disabled, ) + stall_stop = st.checkbox( + "Stop early if the model stalls", + value=False, + disabled=disabled, + help=( + "End the search after N consecutive no-progress iterations " + "(the model repeating an invalid action or applying no change). " + "Saves iterations when a model is stuck. Leave unchecked for " + "comparable runs that always reach Max iterations." + ), + ) + stall_limit = st.number_input( + "Stop after N no-progress iterations", + min_value=2, max_value=20, value=5, step=1, + disabled=disabled or not stall_stop, + ) + os.environ["AGENTIGRID_STALL_LIMIT"] = str(int(stall_limit)) if stall_stop else "0" mpi_disabled = disabled or application not in ("scopflow", "sopflow") mpi_np = st.number_input( "MPI processes", @@ -797,15 +815,23 @@ def render_live_monitor(): icon = _iteration_icon(entry) obj = entry.get("objective_value") is_pflow_live = st.session_state.get("current_application") == "pflow" - if is_pflow_live: - obj_str = "analysis" if obj is not None else "FAILED" + # An entry with no objective is only "FAILED" if it was a real solve + # that did not converge. Analysis/complete/sweep/contingency/explore + # markers and discarded proposals carry no cost — they are not failures. + cs = entry.get("convergence_status") or "" + _no_solve = cs in NON_SOLVE_STATUSES or entry.get("mode") == "discarded" + if obj is not None: + obj_str = "analysis" if is_pflow_live else f"${obj:,.2f}" + elif _no_solve: + obj_str = "" # no cost, but not a failure — omit the segment else: - obj_str = f"${obj:,.2f}" if obj is not None else "FAILED" + obj_str = "FAILED" # a solve that genuinely did not converge elapsed = entry.get("sim_elapsed") time_str = f"{elapsed:.1f}s" if elapsed is not None else "—" + _cost_seg = f" — {obj_str}" if obj_str else "" label = ( f"{icon} Iteration {entry['iteration']}: " - f"{entry['description']} — {obj_str} ({time_str})" + f"{entry['description']}{_cost_seg} ({time_str})" ) with st.expander(label): @@ -933,7 +959,9 @@ def render_live_monitor(): # Progress Stats st.markdown("---") - n_iters = len(st.session_state.iteration_log) + _its = {e.get("iteration") for e in st.session_state.iteration_log} + _n_agent = sum(1 for i in _its if isinstance(i, int) and i >= 1) + n_iters = f"{_n_agent} + base case" if 0 in _its else str(_n_agent) feasible_count = sum( 1 for e in st.session_state.iteration_log if e.get("feasible") ) @@ -1329,7 +1357,7 @@ def _render_overview_tab(session): mc1, mc2, mc3, mc4, mc5 = st.columns(5) mc1.metric("Application", f"{session.application}") - mc2.metric("Iterations", f"{stats['total_iterations']}") + mc2.metric("Iterations", format_iteration_count(stats)) mc3.metric("Duration", duration_str) mc4.metric("Termination", session.termination_reason) mc5.metric("Tokens", f"{total_tokens:,}" if total_tokens > 0 else "—") diff --git a/agentigrid/launcher/config_builder.py b/agentigrid/launcher/config_builder.py index 4e13f207..a410bb61 100644 --- a/agentigrid/launcher/config_builder.py +++ b/agentigrid/launcher/config_builder.py @@ -253,7 +253,7 @@ def load_example_goals() -> list[dict[str, str]]: # Default model names per backend DEFAULT_MODELS: dict[str, str] = { - "anthropic": "claude-sonnet-4-20250514", + "anthropic": "claude-sonnet-4-6", "openai": "gpt-4o", "ollama": "qwen2.5:7b", "ollama-cloud": "qwen2.5:7b", diff --git a/agentigrid/launcher/report_generator.py b/agentigrid/launcher/report_generator.py index 22076181..d6a7f2e2 100644 --- a/agentigrid/launcher/report_generator.py +++ b/agentigrid/launcher/report_generator.py @@ -23,7 +23,7 @@ from reportlab.pdfbase.ttfonts import TTFont from agentigrid.engine.agent_loop import SearchSession -from agentigrid.engine.journal import is_solve_iteration +from agentigrid.engine.journal import format_iteration_count, is_solve_iteration from agentigrid.parsers.opflow_results import OPFLOWResult try: @@ -599,7 +599,7 @@ def _build_executive_summary( "is not deliverable for these losses (required reserve is mis-located)." ) res_lines.extend([ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", (f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A"), @@ -653,7 +653,7 @@ def _build_executive_summary( relief_line += f"; {unresolved_count} unresolved." cont_lines.append(relief_line) cont_lines.extend([ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", (f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A"), @@ -691,7 +691,7 @@ def _build_executive_summary( _headline = f"Sweep complete: {n_feas} of {n_cand} candidate buses are feasible." sweep_lines = [ _headline, - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Duration: {duration.total_seconds():.0f}s", f"Termination: {session.termination_reason}", f"Token usage: {total_tokens:,}" if total_tokens > 0 else "Token usage: N/A", @@ -715,7 +715,7 @@ def _build_executive_summary( 1 for e in solve_entries if e.feasibility_detail == "marginal" ) lines = [ - f"Total iterations: {stats['total_iterations']}", + f"Iterations: {format_iteration_count(stats)}", f"Feasible solutions: {solve_feasible}", f"Infeasible: {solve_infeasible}", ] @@ -1539,8 +1539,9 @@ def _build_convergence_section( ) # Auto-generated text - n = stats["total_iterations"] - text = f"The search ran for {n} iterations." + n = stats.get("llm_iterations", stats["total_iterations"]) + text = (f"The search ran for {n} iteration(s) after the base case." + if stats.get("has_base_case") else f"The search ran for {n} iteration(s).") if stats["best_objective"] is not None: base_entry = session.journal.entries[0] if session.journal.entries else None if base_entry and base_entry.objective_value and base_entry.objective_value != 0: diff --git a/agentigrid/tests/README.md b/agentigrid/tests/README.md index d9aec734..261752ae 100644 --- a/agentigrid/tests/README.md +++ b/agentigrid/tests/README.md @@ -16,6 +16,12 @@ Tests are grouped by the part of AgentiGrid they cover: | `e2e/` | End-to-end runs through the agent loop | | `fixtures/` | Shared sample files (e.g. `sample_opflow_output.txt`) | +`fixtures/sample_opflow_output.txt` is OPFLOW output for `case_ACTIVSg200.m` as +rewritten by AgentiGrid; `results/test_opflow_parser.py` checks values from it. +To regenerate it (needs the `opflow` binary), run the live executor test with +`AGENTIGRID_REGENERATE_FIXTURES=1 python -m pytest tests/runner/test_executor.py -k test_opflow_run` +and update the expected values in `results/test_opflow_parser.py`. + Run the fast tests from the AgentiGrid root: ```bash diff --git a/agentigrid/tests/agent_loop/test_agent_loop.py b/agentigrid/tests/agent_loop/test_agent_loop.py index 2742af12..32222f4d 100644 --- a/agentigrid/tests/agent_loop/test_agent_loop.py +++ b/agentigrid/tests/agent_loop/test_agent_loop.py @@ -254,6 +254,152 @@ def test_invalid_json_then_valid(self, tmp_path: Path): assert len(session.journal) == 3 # base + modify + complete assert session.termination_reason == "completed" + def test_discarded_iterations_are_journaled_as_telemetry(self, tmp_path: Path): + """An unparseable action and an unknown action record no entry, but are + kept in journal.discarded_actions (validator-rejection telemetry).""" + cfg = _make_config(tmp_path, max_iterations=10) + responses = [ + _objectives_response(), # objective parser + _make_llm_response(None, raw_text="This is not JSON"), # iter 1 + _make_llm_response({"action": "dance", "reasoning": "x"}), # iter 2 + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}}), # iter 3 + ] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + controller = AgentLoopController(cfg) + session = controller.run(BASE_CASE, "Test discard telemetry") + + assert len(session.journal) == 2 # base + complete + disc = session.journal.discarded_actions + assert [(d["iteration"], d["kind"]) for d in disc] == [(1, "rejected"), (2, "rejected")] + assert "parse JSON" in disc[0]["feedback"] and "Unknown action" in disc[1]["feedback"] + # the rejected proposal itself is kept (raw model output) + assert disc[0]["response"] == "This is not JSON" and disc[0]["response_truncated"] is False + assert '"dance"' in disc[1]["response"] or "dance" in disc[1]["response"] + + def test_api_errors_are_not_counted_as_rejections(self, tmp_path: Path): + cfg = _make_config(tmp_path, max_iterations=10) + api_err = LLMResponse(raw_text="Anthropic API error: 400", json_data=None, json_error="x", + model="m", backend="test", prompt_tokens=None, completion_tokens=None, + api_error=True) + responses = [_objectives_response(), api_err, + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test api error") + disc = session.journal.discarded_actions + assert [(d["iteration"], d["kind"]) for d in disc] == [(1, "api_error")] + assert "response" not in disc[0] # no model response for a failed request + + def test_truncated_output_is_journaled_as_truncated(self, tmp_path: Path): + cfg = _make_config(tmp_path, max_iterations=10) + cut = LLMResponse(raw_text="", json_data=None, json_error="no JSON", model="m", + backend="test", prompt_tokens=10, completion_tokens=4096, + stop_reason="max_tokens", content_types=["thinking"]) + responses = [_objectives_response(), cut, + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test truncation") + d = session.journal.discarded_actions[0] + assert (d["iteration"], d["kind"], d["stop_reason"]) == (1, "truncated", "max_tokens") + assert d["content_types"] == ["thinking"] and d["response"] == "" + assert "output-token limit" in d["feedback"] + + def test_long_discarded_response_is_capped(self, tmp_path: Path): + from agentigrid.engine import agent_loop as al + cfg = _make_config(tmp_path, max_iterations=10) + long_text = "x" * (al.DISCARDED_RESPONSE_MAX + 50) + responses = [_objectives_response(), _make_llm_response(None, raw_text=long_text), + _make_llm_response({"action": "complete", "reasoning": "Done", + "findings": {"summary": "ok"}})] + mock_backend = MockBackend(responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg).run(BASE_CASE, "Test cap") + d = session.journal.discarded_actions[0] + assert len(d["response"]) == al.DISCARDED_RESPONSE_MAX and d["response_truncated"] is True + + def _run(self, tmp_path, responses, goal="Test", max_iterations=10, **ctl_kw): + cfg = _make_config(tmp_path, max_iterations=max_iterations) + mock_backend = MockBackend([_objectives_response()] + responses) + sim_result = _make_sim_result(stdout=_sample_stdout(), success=True) + with patch("agentigrid.engine.agent_loop.create_backend", return_value=mock_backend), \ + patch("agentigrid.engine.agent_loop.SimulationExecutor") as mock_exec_cls: + mock_executor = MagicMock() + mock_executor.run.return_value = sim_result + mock_exec_cls.return_value = mock_executor + return AgentLoopController(cfg, **ctl_kw).run(BASE_CASE, goal) + + _COMPLETE = {"action": "complete", "reasoning": "Done", "findings": {"summary": "ok"}} + + def test_ui_callback_never_replays_a_stale_entry(self, tmp_path: Path): + """A rejected iteration produces a 'discarded' card for that iteration, + never a re-emit of the previous entry, and the run ends without a + final duplicate of the last card.""" + calls = [] + session = self._run( + tmp_path, + [_make_llm_response(None, raw_text="not JSON"), # iter 1: rejected + _make_llm_response({"action": "dance", "reasoning": "x"}), # iter 2: rejected + _make_llm_response(self._COMPLETE)], # iter 3 + on_iteration=lambda it, entry, kind, opf: calls.append((it, kind, entry.mode)), + ) + assert [c[0] for c in calls] == [0, 1, 2, 3] + assert [c[1] for c in calls[1:3]] == ["discarded", "discarded"] + assert all(c[2] == "discarded" for c in calls[1:3]) + assert session.termination_reason == "completed" + # the synthetic cards are UI-only: the journal holds base case + complete + assert len(session.journal) == 2 + + def test_stall_breaker_is_off_by_default(self, tmp_path: Path, monkeypatch): + monkeypatch.delenv("AGENTIGRID_STALL_LIMIT", raising=False) + bad = {"action": "dance", "reasoning": "x"} + session = self._run(tmp_path, [_make_llm_response(bad)] * 3 + [_make_llm_response(self._COMPLETE)]) + assert session.termination_reason == "completed" + assert len(session.journal.discarded_actions) == 3 + + def test_stall_breaker_stops_after_n_no_progress_iterations(self, tmp_path: Path, monkeypatch): + monkeypatch.setenv("AGENTIGRID_STALL_LIMIT", "2") + bad = {"action": "dance", "reasoning": "x"} + session = self._run(tmp_path, [_make_llm_response(bad)] * 3 + [_make_llm_response(self._COMPLETE)]) + assert session.termination_reason == "stalled_no_progress" + assert [d["iteration"] for d in session.journal.discarded_actions] == [1, 2] + + def test_internal_error_in_an_action_is_discarded_and_the_run_continues(self, tmp_path: Path): + with patch.object(AgentLoopController, "_handle_analyze", side_effect=RuntimeError("boom")): + session = self._run( + tmp_path, + [_make_llm_response({"action": "analyze", "query": "max loading", "reasoning": "x"}), + _make_llm_response(self._COMPLETE)], + ) + assert session.termination_reason == "completed" + d = session.journal.discarded_actions[0] + assert (d["iteration"], d["kind"]) == (1, "internal") + assert "RuntimeError: boom" in d["feedback"] + def test_unknown_action(self, tmp_path: Path): """LLM returns unknown action, then completes.""" cfg = _make_config(tmp_path, max_iterations=10) diff --git a/agentigrid/tests/e2e/test_e2e.py b/agentigrid/tests/e2e/test_e2e.py index 5766d4e5..c09cd796 100644 --- a/agentigrid/tests/e2e/test_e2e.py +++ b/agentigrid/tests/e2e/test_e2e.py @@ -113,7 +113,7 @@ def _make_config( data=DataConfig(data_dir=DATA_DIR), llm=LLMConfig( backend="anthropic" if _has_anthropic_key else "openai", - model="claude-sonnet-4-20250514" if _has_anthropic_key else "gpt-4o-mini", + model="claude-sonnet-4-6" if _has_anthropic_key else "gpt-4o-mini", api_key_env="ANTHROPIC_API_KEY" if _has_anthropic_key else "OPENAI_API_KEY", openai_base_url=None, ollama_host="http://localhost:11434", diff --git a/agentigrid/tests/fixtures/sample_opflow_output.txt b/agentigrid/tests/fixtures/sample_opflow_output.txt index cf3656c1..2678c865 100644 --- a/agentigrid/tests/fixtures/sample_opflow_output.txt +++ b/agentigrid/tests/fixtures/sample_opflow_output.txt @@ -5,15 +5,15 @@ This program contains Ipopt, a library for large-scale nonlinear optimization. For more information visit https://github.com/coin-or/Ipopt ****************************************************************************** -This is Ipopt version 3.14.20, running with linear solver ma27. +This is Ipopt version 3.14.14, running with linear solver ma27. -Number of nonzeros in equality constraint Jacobian...: 2826 +Number of nonzeros in equality constraint Jacobian...: 2832 Number of nonzeros in inequality constraint Jacobian.: 1958 -Number of nonzeros in Lagrangian Hessian.............: 1956 +Number of nonzeros in Lagrangian Hessian.............: 1992 -Total number of variables............................: 469 +Total number of variables............................: 475 variables with only lower bounds: 0 - variables with lower and upper bounds: 270 + variables with lower and upper bounds: 276 variables with only upper bounds: 0 Total number of equality constraints.................: 400 Total number of inequality constraints...............: 490 @@ -22,52 +22,50 @@ Total number of inequality constraints...............: 490 inequality constraints with only upper bounds: 490 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 0 4.0417248e+04 3.70e+00 4.65e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 - 1 3.9034986e+04 3.40e+00 4.25e+01 -1.0 2.83e+00 - 1.39e-01 8.08e-02f 1 - 2 3.6673124e+04 2.88e+00 3.56e+01 -1.0 7.20e+00 - 4.19e-01 1.48e-01f 1 - 3 3.1578046e+04 1.80e+00 1.23e+02 -1.0 2.11e+01 - 8.80e-01 3.49e-01f 1 - 4 3.0599807e+04 1.52e+00 1.10e+02 -1.0 2.12e+01 - 4.46e-01 1.53e-01f 1 - 5 2.9699291e+04 1.26e+00 9.78e+01 -1.0 1.84e+01 - 4.37e-01 1.71e-01f 1 - 6 2.9309885e+04 1.15e+00 9.04e+01 -1.0 1.59e+01 - 6.85e-01 8.97e-02f 1 - 7 2.8645960e+04 8.46e-01 7.05e+01 -1.0 3.67e+00 - 2.11e-03 2.64e-01f 1 - 8 2.7638522e+04 2.53e-02 7.27e+01 -1.0 3.56e+00 - 9.30e-01 1.00e+00f 1 - 9 2.7640850e+04 8.98e-04 7.46e-01 -1.0 7.87e-01 - 1.00e+00 1.00e+00h 1 + 0 4.0417248e+04 3.70e+00 6.61e+01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 + 1 3.8873587e+04 3.57e+00 6.36e+01 -1.0 4.46e+00 - 4.92e-02 3.53e-02f 1 + 2 3.6809680e+04 3.38e+00 5.88e+01 -1.0 4.50e+00 - 1.55e-01 5.19e-02f 1 + 3 3.4170482e+04 3.08e+00 4.94e+01 -1.0 3.95e+00 - 2.60e-01 8.73e-02f 1 + 4 3.2530838e+04 2.69e+00 2.99e+01 -1.0 9.39e+00 - 3.74e-01 1.23e-01f 1 + 5 2.9302664e+04 1.78e+00 8.68e+01 -1.0 2.07e+01 - 9.47e-01 3.17e-01f 1 + 6 2.8914676e+04 1.56e+00 7.90e+01 -1.0 1.48e+01 - 3.97e-01 1.26e-01f 1 + 7 2.8108655e+04 6.74e-01 5.46e+01 -1.0 5.25e+00 - 9.65e-01 5.69e-01f 1 + 8 2.7648421e+04 1.63e-02 2.21e+01 -1.0 1.66e+00 - 8.62e-01 1.00e+00f 1 + 9 2.7654216e+04 1.36e-03 1.97e-01 -1.0 3.99e-01 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 10 2.7585786e+04 1.54e-04 9.33e-02 -1.7 3.31e-01 - 1.00e+00 1.00e+00f 1 - 11 2.7569878e+04 1.44e-03 5.82e-02 -2.5 6.72e-02 - 9.88e-01 1.00e+00f 1 - 12 2.7563905e+04 1.32e-02 2.35e-02 -2.5 1.04e-01 - 1.00e+00 1.00e+00h 1 - 13 2.7564551e+04 3.76e-05 1.44e-04 -2.5 2.12e-02 - 1.00e+00 1.00e+00h 1 - 14 2.7558777e+04 6.47e-03 6.79e-02 -3.8 7.82e-02 - 8.63e-01 1.00e+00f 1 - 15 2.7557880e+04 4.68e-03 1.05e-01 -3.8 7.97e-02 - 1.00e+00 3.39e-01h 1 - 16 2.7557754e+04 5.40e-04 4.26e-01 -3.8 1.03e-01 - 1.00e+00 8.88e-01h 1 - 17 2.7557736e+04 9.85e-06 2.13e-04 -3.8 2.55e-02 - 1.00e+00 1.00e+00h 1 - 18 2.7557602e+04 1.37e-05 1.54e-01 -5.7 3.45e-02 - 9.15e-01 7.81e-01h 1 - 19 2.7557572e+04 3.77e-06 1.09e-02 -5.7 9.25e-03 - 1.00e+00 9.65e-01h 1 + 10 2.7574726e+04 1.60e-04 2.22e+00 -2.5 2.75e-01 - 8.77e-01 1.00e+00f 1 + 11 2.7569519e+04 2.59e-03 1.09e-02 -2.5 7.33e-02 - 1.00e+00 1.00e+00f 1 + 12 2.7560217e+04 1.93e-02 1.30e-01 -3.8 1.26e-01 - 7.37e-01 1.00e+00h 1 + 13 2.7557763e+04 1.06e-02 2.11e-02 -3.8 1.04e-01 - 9.63e-01 6.25e-01h 1 + 14 2.7557837e+04 1.74e-03 6.51e-01 -3.8 9.82e-02 - 1.00e+00 8.38e-01h 1 + 15 2.7557760e+04 1.86e-05 4.99e-04 -3.8 2.51e-02 - 1.00e+00 1.00e+00h 1 + 16 2.7557607e+04 1.53e-05 1.57e-01 -5.7 3.40e-02 - 9.19e-01 7.80e-01h 1 + 17 2.7557573e+04 3.73e-06 1.16e-02 -5.7 9.12e-03 - 1.00e+00 9.63e-01h 1 + 18 2.7557573e+04 1.95e-06 9.06e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 + 19 2.7557573e+04 6.41e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls - 20 2.7557572e+04 1.94e-06 8.78e-03 -5.7 1.36e-03 - 1.00e+00 5.00e-01h 2 - 21 2.7557572e+04 6.42e-07 2.58e-06 -5.7 1.07e-03 - 1.00e+00 1.00e+00h 1 - 22 2.7557571e+04 2.07e-07 1.75e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 - 23 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 + 20 2.7557571e+04 2.07e-07 1.73e-04 -7.0 6.18e-04 - 9.89e-01 1.00e+00h 1 + 21 2.7557571e+04 5.19e-08 2.16e-07 -7.0 3.05e-04 - 1.00e+00 1.00e+00h 1 -Number of Iterations....: 23 +Number of Iterations....: 21 (scaled) (unscaled) -Objective...............: 1.1868829499149242e+03 2.7557570559622669e+04 -Dual infeasibility......: 2.1624874764514693e-07 5.0209585722739846e-06 -Constraint violation....: 5.1921755382135226e-08 5.1921755382135226e-08 -Variable bound violation: 1.1735202365414921e-08 1.1735202365414921e-08 -Complementarity.........: 2.0827918551588903e-07 4.8359177721495380e-06 -Overall NLP error.......: 2.1624874764514693e-07 5.0209585722739846e-06 +Objective...............: 1.1868829484688854e+03 2.7557570526047901e+04 +Dual infeasibility......: 2.1611411147490891e-07 5.0178325304334832e-06 +Constraint violation....: 5.1889360219194158e-08 5.1889360219194158e-08 +Variable bound violation: 1.2335941157459729e-08 1.2335941157459729e-08 +Complementarity.........: 2.0818450105630093e-07 4.8337193467056586e-06 +Overall NLP error.......: 2.1611411147490891e-07 5.0178325304334832e-06 -Number of objective function evaluations = 25 -Number of objective gradient evaluations = 24 -Number of equality constraint evaluations = 25 -Number of inequality constraint evaluations = 25 -Number of equality constraint Jacobian evaluations = 24 -Number of inequality constraint Jacobian evaluations = 24 -Number of Lagrangian Hessian evaluations = 23 -Total seconds in IPOPT = 0.063 +Number of objective function evaluations = 23 +Number of objective gradient evaluations = 22 +Number of equality constraint evaluations = 23 +Number of inequality constraint evaluations = 23 +Number of equality constraint Jacobian evaluations = 22 +Number of inequality constraint Jacobian evaluations = 22 +Number of Lagrangian Hessian evaluations = 21 +Total seconds in IPOPT = 0.021 EXIT: Optimal Solution Found. ============================================================= @@ -551,7 +549,7 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 51 1 COAL 1.36 0.53 1.36 4.53 -0.55 2.11 52 1 COAL 1.36 0.65 1.36 4.53 -0.55 2.11 53 1 COAL 2.72 0.70 2.72 9.07 -1.11 4.23 -65 1 COAL 86.50 -4.97 86.50 86.50 -21.66 32.04 +65 1 WIND 86.50 -4.97 0.00 86.50 -21.66 32.04 67 1 COAL 1.41 -0.57 1.41 4.70 -0.57 2.19 68 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 69 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 @@ -559,39 +557,39 @@ Gen Status Fuel Pg Qg Pmin Pmax Qmin Qmax 71 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 72 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 73 1 COAL 8.38 -3.41 8.38 27.92 -3.41 13.01 -76 1 COAL 1.20 0.52 1.20 4.00 -0.44 2.04 -77 1 COAL 0.72 0.26 0.72 2.40 -0.27 1.22 -78 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -79 0 COAL 0.00 0.00 5.40 18.00 -2.00 9.16 -90 1 COAL 0.96 -0.36 0.96 3.20 -0.36 1.63 -91 1 COAL 1.50 -0.55 1.50 5.00 -0.55 2.55 -92 0 COAL 0.00 0.00 1.89 6.30 -0.70 3.21 +76 1 NG 1.20 0.52 1.20 4.00 -0.44 2.04 +77 1 NG 0.72 0.26 0.72 2.40 -0.27 1.22 +78 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +79 0 NG 0.00 0.00 5.40 18.00 -2.00 9.16 +90 1 NG 0.96 -0.36 0.96 3.20 -0.36 1.63 +91 1 NG 1.50 -0.55 1.50 5.00 -0.55 2.55 +92 0 NG 0.00 0.00 1.89 6.30 -0.70 3.21 94 1 COAL 5.40 -2.20 5.40 18.00 -2.20 8.39 -104 1 COAL 67.60 0.80 67.60 67.60 -14.26 21.09 -105 1 COAL 154.80 6.45 154.80 154.80 -28.51 42.17 -114 1 COAL 1.40 0.30 1.40 1.40 -0.24 0.36 -115 1 COAL 133.50 23.61 133.50 133.50 -21.60 31.95 +104 1 WIND 67.60 0.80 0.00 67.60 -14.26 21.09 +105 1 WIND 154.80 6.45 0.00 154.80 -28.51 42.17 +114 1 WIND 1.40 0.30 0.00 1.40 -0.24 0.36 +115 1 WIND 133.50 23.61 0.00 133.50 -21.60 31.95 125 1 COAL 39.02 6.39 39.02 130.05 -15.87 60.60 126 1 COAL 39.02 16.14 39.02 130.05 -15.87 60.60 127 1 COAL 39.02 22.98 39.02 130.05 -15.87 60.60 135 1 COAL 133.92 26.66 133.92 446.40 -54.46 208.02 136 1 COAL 133.92 25.72 133.92 446.40 -54.46 208.02 -147 1 COAL 92.40 11.12 92.40 92.40 -14.47 21.41 +147 1 WIND 92.40 11.12 0.00 92.40 -14.47 21.41 151 1 COAL 1.62 -0.61 1.62 5.40 -0.66 2.52 152 1 COAL 23.17 -5.34 23.17 77.22 -9.42 35.99 153 1 COAL 23.17 -1.87 23.17 77.22 -9.42 35.99 154 1 COAL 23.17 -5.54 23.17 77.22 -9.42 35.99 155 1 COAL 23.17 -5.55 23.17 77.22 -9.42 35.99 -161 0 COAL 0.00 0.00 41.58 138.60 -15.38 70.55 -164 0 COAL 0.00 0.00 3.60 12.00 -1.33 6.11 -165 0 COAL 0.00 0.00 7.80 26.00 -2.89 13.23 -166 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -167 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 -168 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -169 0 COAL 0.00 0.00 2.82 9.40 -1.04 4.78 -170 1 COAL 2.82 -1.04 2.82 9.40 -1.04 4.78 +161 0 NG 0.00 0.00 41.58 138.60 -15.38 70.55 +164 0 NG 0.00 0.00 3.60 12.00 -1.33 6.11 +165 0 NG 0.00 0.00 7.80 26.00 -2.89 13.23 +166 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +167 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 +168 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +169 0 NG 0.00 0.00 2.82 9.40 -1.04 4.78 +170 1 NG 2.82 -1.04 2.82 9.40 -1.04 4.78 182 1 COAL 5.25 0.88 5.25 17.50 -2.14 8.16 183 1 COAL 7.98 1.99 7.98 26.60 -3.25 12.40 -189 1 COAL 383.40 -46.67 170.75 569.15 -46.67 209.45 -196 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 -197 0 COAL 0.00 0.00 20.25 67.50 -7.49 34.36 +189 1 NUCLEAR 383.40 -46.67 170.75 569.15 -46.67 209.45 +196 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 +197 0 NG 0.00 0.00 20.25 67.50 -7.49 34.36 diff --git a/agentigrid/tests/grid_files/test_matpower_cell_terminators.py b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py new file mode 100644 index 00000000..f845cf17 --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_cell_terminators.py @@ -0,0 +1,72 @@ +"""Regression: rewritten cases must keep the "};" terminator on cell arrays. + +ExaGO's MATPOWER reader (src/ps/psreaddata.cpp) finds the end of mpc.genfuel +by searching for the literal "};". The writer used to emit a bare "}", so ExaGO +silently ignored the fuel types of every case AgentiGrid rewrote: ramp rates +stayed 0 (SCOPFLOW could not redispatch in contingencies and reported local +infeasibility on ACTIVSg200) and wind/solar units stayed fixed at Pmax. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from agentigrid.parsers.matpower_model import MATNetwork +from agentigrid.parsers.matpower_parser import parse_matpower +from agentigrid.parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + + +def _exago_genfuel_entries(path: Path) -> list[str] | None: + """Mimic psreaddata.cpp: genfuel starts after the 'mpc.genfuel' line and + ends at the first later line containing '};'. None = end never found.""" + lines = path.read_text().splitlines() + start = end = None + for i, line in enumerate(lines): + if start is None and "mpc.genfuel" in line: + start = i + 1 + elif start is not None and "};" in line: + end = i + break + if start is None or end is None: + return None + return [m.group(1) for l in lines[start:end] if (m := re.search(r"'([^']*)'", l))] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_roundtrip_keeps_genfuel_readable_by_exago(tmp_path): + original = _exago_genfuel_entries(ACTIVSG200) + assert original and "wind" in original + out = tmp_path / "rt.m" + write_matpower(parse_matpower(ACTIVSG200), out) + assert _exago_genfuel_entries(out) == original + # every cell array closes with "};", every matrix with "];" + closers = [l.strip() for l in out.read_text().splitlines() if l.strip()[:1] in ("}", "]")] + assert closers and all(c in ("};", "];") for c in closers) + + +def test_writer_terminates_legacy_raw_sections(tmp_path): + """Networks parsed before the fix (e.g. restored sessions) stored '}'.""" + net = MATNetwork(casename="c", version="2", baseMVA=100.0, buses=[], generators=[], + branches=[], gencost=[], header_comments="function mpc = c\n", + extra_sections={"genfuel": "mpc.genfuel = {\n\t'wind';\n}", + "other": "mpc.other = [\n\t1 2;\n]"}) + out = tmp_path / "c.m" + write_matpower(net, out) + text = out.read_text() + assert "\n};\n" in text and "\n];\n" in text + assert _exago_genfuel_entries(out) == ["wind"] + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_double_roundtrip_is_stable(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert "};;" not in b.read_text() and "];;" not in b.read_text() + assert _exago_genfuel_entries(b) == _exago_genfuel_entries(ACTIVSG200) diff --git a/agentigrid/tests/grid_files/test_matpower_hardening.py b/agentigrid/tests/grid_files/test_matpower_hardening.py new file mode 100644 index 00000000..88f71b3c --- /dev/null +++ b/agentigrid/tests/grid_files/test_matpower_hardening.py @@ -0,0 +1,279 @@ +"""MATPOWER reader/writer hardening: no silent misreads, stable round trips. + +Covers five defects found while specifying the parser: + 1. malformed rows were skipped with only a log warning (a bus could vanish); + 2. valid-but-unsupported MATLAB syntax was misread (two rows on one line + became one garbage row; comma-separated rows were all dropped); + 3. the writer crashed on Inf/NaN, which the reader accepts; + 4. every read/write round trip added another header line; + 5. MATPOWER case version '1' was accepted although ExaGO reads version 2. +""" + +from __future__ import annotations + +import copy +import math +from pathlib import Path + +import pytest + +from agentigrid.parsers.matpower_model import Branch, Bus, GenCost, Generator, MATNetwork +from agentigrid.parsers.matpower_parser import parse_matpower +from agentigrid.parsers.matpower_writer import write_matpower + +ROOT = Path(__file__).resolve().parents[2] +ACTIVSG200 = ROOT.parent / "datafiles" / "case_ACTIVSg200.m" + +BUS = "\t1\t3\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;\n\t2\t1\t90\t30\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" +GEN = "\t1\t0\t0\t300\t-300\t1\t100\t1\t250\t10;" +BRANCH = "\t1\t2\t0.01\t0.1\t0\t100\t100\t100\t0\t0\t1\t-360\t360;" +GENCOST = "\t2\t0\t0\t3\t0.11\t5\t150;" + + +def _case(*, version="2", bus=BUS, gen=GEN, branch=BRANCH, gencost=GENCOST, extra="", + drop: tuple[str, ...] = ()) -> str: + parts = { + "head": "function mpc = tiny\n%TINY test case\n\n%% MATPOWER Case Format : Version 2\n", + "version": f"mpc.version = '{version}';\n", + "base": "\n%% system MVA base\nmpc.baseMVA = 100;\n", + "bus": f"\n%% bus data\nmpc.bus = [\n{bus}\n];\n", + "gen": f"\n%% generator data\nmpc.gen = [\n{gen}\n];\n", + "branch": f"\n%% branch data\nmpc.branch = [\n{branch}\n];\n", + "gencost": f"\n%% generator cost data\nmpc.gencost = [\n{gencost}\n];\n", + "extra": f"\n{extra}\n" if extra else "", + } + return "".join(v for k, v in parts.items() if k not in drop) + + +def _write(tmp_path: Path, text: str, name: str = "c.m", crlf: bool = False) -> Path: + p = tmp_path / name + data = text.replace("\n", "\r\n") if crlf else text + p.write_bytes(data.encode("utf-8")) + return p + + +# --- 5. version ------------------------------------------------------------ + +def test_version_1_rejected(tmp_path): + with pytest.raises(ValueError, match="version"): + parse_matpower(_write(tmp_path, _case(version="1"))) + + +def test_missing_version_defaults_to_2(tmp_path): + text = _case().replace("mpc.version = '2';\n", "") + assert parse_matpower(_write(tmp_path, text)).version == "2" + + +# --- 1./2. malformed rows and unsupported syntax --------------------------- + +@pytest.mark.parametrize( + "kwargs, section", + [ + ({"bus": BUS.replace("\n", " ")}, "bus"), # two rows, one line + ({"bus": BUS.replace("\t", ",")}, "bus"), # commas + ({"gen": "\t1\t0\t0\t300\t-300 ...\n\t1\t100\t1\t250\t10;"}, "gen"), # continuation + ({"bus": BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;"}, "bus"), # non-numeric + ({"bus": "\t1\t3\t0\t0;"}, "bus"), # too few columns + ({"gen": "\t1\t0\t0;"}, "gen"), + ({"branch": "\t1\t2\t0.01;"}, "branch"), + ({"gencost": "\t2\t0\t0;"}, "gencost"), + ({"bus": BUS + "\n\t3" + "\t1" * 17 + ";"}, "bus"), # 18 bus columns + ({"bus": BUS.replace("\t1\t3\t", "\t1.5\t3\t", 1)}, "bus"), # non-integral bus_i + ({"gen": GEN.replace("\t100\t1\t", "\t100\tInf\t")}, "gen"), # Inf status + ({"branch": BRANCH.replace("\t0\t1\t-360", "\t0\tNaN\t-360")}, "branch"), # NaN status + ], + ids=["rows-one-line", "commas", "continuation", "non-numeric", "short-bus", "short-gen", + "short-branch", "short-gencost", "bus-18-cols", "int-col-1.5", "int-col-inf", + "int-col-nan"], +) +def test_malformed_input_raises_naming_section(tmp_path, kwargs, section): + with pytest.raises(ValueError, match=rf"mpc\.{section}\b"): + parse_matpower(_write(tmp_path, _case(**kwargs))) + + +def test_non_numeric_row_reports_row_number(tmp_path): + bad = BUS + "\n\t3\tX\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;" + with pytest.raises(ValueError, match=r"mpc\.bus row 3"): + parse_matpower(_write(tmp_path, _case(bus=bad))) + + +def test_unterminated_block_raises(tmp_path): + text = _case(drop=("gencost",)).replace("\n];\n\n%% branch data", "\n\n%% branch data") + with pytest.raises(ValueError, match="terminated"): + parse_matpower(_write(tmp_path, text)) + + +def test_missing_bus_section_raises(tmp_path): + with pytest.raises(ValueError, match="mpc.bus"): + parse_matpower(_write(tmp_path, _case(drop=("bus",)))) + + +def test_missing_gen_and_branch_give_empty_lists(tmp_path): + net = parse_matpower(_write(tmp_path, _case(drop=("gen", "branch", "gencost")))) + assert net.generators == [] and net.branches == [] and net.gencost == [] + + +def test_partial_bus_solution_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5;", 1)))) + b = net.buses[0] + assert b.lam_P == 6.5 and b.lam_Q is None and b.mu_Vmax is None and b.mu_Vmin is None + + +def test_integral_floats_accepted_in_integer_columns(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("\t1\t3\t", "\t1.0\t3.0\t", 1)))) + assert net.buses[0].bus_i == 1 and isinstance(net.buses[0].bus_i, int) + + +# --- line endings ------------------------------------------------------------ + +def test_crlf_reads_like_lf_and_writer_emits_lf(tmp_path): + lf = parse_matpower(_write(tmp_path, _case(), "lf.m")) + crlf = parse_matpower(_write(tmp_path, _case(), "crlf.m", crlf=True)) + assert crlf.buses == lf.buses and crlf.header_comments == lf.header_comments + out = tmp_path / "out.m" + write_matpower(crlf, out) + assert b"\r" not in out.read_bytes() + + +# --- 3. Inf / NaN ---------------------------------------------------------- + +def test_inf_nan_round_trip(tmp_path): + net = parse_matpower(_write(tmp_path, _case())) + net.branches[0].rateA = math.inf + net.generators[0].Qmin = -math.inf + net.buses[1].Pd = math.nan + out = tmp_path / "out.m" + write_matpower(net, out) + back = parse_matpower(out) + assert back.branches[0].rateA == math.inf + assert back.generators[0].Qmin == -math.inf + assert math.isnan(back.buses[1].Pd) + + +def test_solution_columns_keep_precision(tmp_path): + net = parse_matpower(_write(tmp_path, _case(bus=BUS.replace("0.9;", "0.9\t6.5\t0\t1e-07\t0.000123456789;", 1)))) + out = tmp_path / "out.m" + write_matpower(net, out) + b = parse_matpower(out).buses[0] + assert math.isclose(b.mu_Vmax, 1e-07, rel_tol=1e-9) + assert math.isclose(b.mu_Vmin, 0.000123456789, rel_tol=1e-9) + + +# --- 4. idempotent writing --------------------------------------------------- + +EXTRAS = ( + "%% generator fuel type\nmpc.genfuel = {\n\t'coal';\n};\n\n" + "mpc.areas = [\n\t1\t1;\n];" +) + + +def test_round_trip_text_is_idempotent(tmp_path): + src = _write(tmp_path, _case(extra=EXTRAS), crlf=True) + a, b, c = (tmp_path / n for n in ("a.m", "b.m", "c.m")) + write_matpower(parse_matpower(src), a) + write_matpower(parse_matpower(a), b) + write_matpower(parse_matpower(b), c) + assert a.read_text() == b.read_text() == c.read_text() + assert a.read_text().count("MATPOWER Case Format") == 1 + + +@pytest.mark.skipif(not ACTIVSG200.exists(), reason="ExaGO datafiles not present") +def test_real_case_round_trip_is_idempotent(tmp_path): + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(parse_matpower(ACTIVSG200), a) + write_matpower(parse_matpower(a), b) + assert a.read_text() == b.read_text() + + +def test_section_comments_stay_with_their_section(tmp_path): + out = tmp_path / "out.m" + write_matpower(parse_matpower(_write(tmp_path, _case(extra=EXTRAS))), out) + text = out.read_text() + assert text.index("%% generator data") < text.index("mpc.gen =") + assert text.index("];", text.index("mpc.bus =")) < text.index("%% generator data") + assert text.index("%% generator fuel type") < text.index("mpc.genfuel =") + + +# --- casename / function line ------------------------------------------------ + +def _net(header: str, casename: str = "mine") -> MATNetwork: + return MATNetwork( + casename=casename, version="2", baseMVA=100.0, + buses=[Bus(1, 3, 0.0, 0.0, 0.0, 0.0, 1, 1.0, 0.0, 230.0, 1, 1.1, 0.9)], + generators=[Generator(1, 0.0, 0.0, 300.0, -300.0, 1.0, 100.0, 1, 250.0, 10.0)], + branches=[Branch(1, 1, 0.01, 0.1, 0.0, 100.0, 100.0, 100.0, 0.0, 0.0, 1, -360.0, 360.0)], + gencost=[GenCost(2, 0.0, 0.0, 3, [0.11, 5.0, 150.0])], + header_comments=header, + ) + + +def test_casename_wins_over_header_function_line(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net("function mpc = other\n%comment\n", casename="mine"), out) + text = out.read_text() + assert "function mpc = mine" in text and "function mpc = other" not in text + assert text.count("function mpc") == 1 + assert parse_matpower(out).casename == "mine" + + +def test_empty_header_writes_function_line_first(tmp_path): + out = tmp_path / "out.m" + write_matpower(_net(""), out) + assert out.read_text().splitlines()[0] == "function mpc = mine" + assert parse_matpower(out).casename == "mine" + + +def test_writer_does_not_mutate_its_input(tmp_path): + net = parse_matpower(_write(tmp_path, _case(extra=EXTRAS))) + before = copy.deepcopy(net) + write_matpower(net, tmp_path / "out.m") + assert net == before + + +# --- number formatting and baseMVA (found by property-based testing) --------- + +@pytest.mark.parametrize( + "value", + [9999999999.5, 123456789012.345, 1e15, 1e16, 0.1 + 0.2, 5e-324, + 1.7976931345e308, -1.7976931348623157e308], + ids=["rounds-to-int", "11-digits", "1e15", "1e16", "0.3", "subnormal", + "near-max", "-max"], +) +def test_number_round_trip_is_close_finite_and_idempotent(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.buses[1].Pd = value + a, b = tmp_path / "a.m", tmp_path / "b.m" + write_matpower(net, a) + back = parse_matpower(a) + assert math.isfinite(back.buses[1].Pd) + assert math.isclose(back.buses[1].Pd, value, rel_tol=1e-9, abs_tol=1e-12) + write_matpower(back, b) + assert a.read_text() == b.read_text() + + +@pytest.mark.parametrize( + "text, expected", + [("1e2", 100.0), ("100.0", 100.0), ("-1", -1.0), ("1e+16", 1e16), ("Inf", math.inf)], +) +def test_basemva_accepts_any_numeric_literal(tmp_path, text, expected): + net = parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", f"mpc.baseMVA = {text};"))) + assert net.baseMVA == expected + + +def test_basemva_in_a_comment_is_ignored(tmp_path): + text = _case().replace("%TINY test case\n", "%TINY test case\n% mpc.baseMVA = 5;\n") + assert parse_matpower(_write(tmp_path, text)).baseMVA == 100.0 + + +def test_basemva_non_numeric_raises(tmp_path): + with pytest.raises(ValueError, match="baseMVA"): + parse_matpower(_write(tmp_path, _case().replace("mpc.baseMVA = 100;", "mpc.baseMVA = abc;"))) + + +@pytest.mark.parametrize("value", [1e16, -1.0, 1.7976931345e308]) +def test_basemva_round_trip(tmp_path, value): + net = parse_matpower(_write(tmp_path, _case())) + net.baseMVA = value + out = tmp_path / "out.m" + write_matpower(net, out) + assert math.isclose(parse_matpower(out).baseMVA, value, rel_tol=1e-9) diff --git a/agentigrid/tests/journal/test_iteration_count.py b/agentigrid/tests/journal/test_iteration_count.py new file mode 100644 index 00000000..2e3c6d54 --- /dev/null +++ b/agentigrid/tests/journal/test_iteration_count.py @@ -0,0 +1,51 @@ +"""Iteration count shown to users: agent iterations plus the base case.""" + +from __future__ import annotations + +from agentigrid.engine.journal import SearchJournal, format_iteration_count + +from tests.journal.test_journal import _make_entry + + +def _journal(iterations, discarded=()): + j = SearchJournal() + for i in iterations: + j.add_entry(_make_entry(i)) + j.discarded_actions = [{"iteration": i, "kind": "rejected"} for i in discarded] + return j + + +def test_base_case_is_not_counted_as_an_agent_iteration(): + stats = _journal([0, 1, 2]).summary_stats() + assert stats["total_iterations"] == 3 # unchanged: journal entries + assert stats["llm_iterations"] == 2 + assert stats["has_base_case"] is True + assert format_iteration_count(stats) == "2 + base case" + + +def test_rejected_iterations_still_count(): + # max 4: iteration 3 was rejected (no journal entry, kept in discarded_actions) + stats = _journal([0, 1, 2, 4], discarded=[3]).summary_stats() + assert stats["total_iterations"] == 4 + assert stats["llm_iterations"] == 4 + assert format_iteration_count(stats) == "4 + base case" + + +def test_several_entries_for_one_iteration_count_once(): + stats = _journal([0, 1, 1, 2]).summary_stats() + assert stats["llm_iterations"] == 2 + + +def test_no_base_case(): + stats = _journal([1, 2]).summary_stats() + assert format_iteration_count(stats) == "2" + + +def test_old_stats_without_new_keys_fall_back(): + assert format_iteration_count({"total_iterations": 5}) == "5" + + +def test_empty_journal(): + stats = SearchJournal().summary_stats() + assert stats["llm_iterations"] == 0 and stats["has_base_case"] is False + assert format_iteration_count(stats) == "0" diff --git a/agentigrid/tests/llm/test_backends.py b/agentigrid/tests/llm/test_backends.py index 2111ecfe..97ed40b4 100644 --- a/agentigrid/tests/llm/test_backends.py +++ b/agentigrid/tests/llm/test_backends.py @@ -233,7 +233,7 @@ class TestAnthropicLive: def test_simple_prompt(self): cfg = _dummy_config( backend="anthropic", - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-6", api_key_env="ANTHROPIC_API_KEY", ) backend = create_backend(cfg) diff --git a/agentigrid/tests/llm/test_prompt_cache.py b/agentigrid/tests/llm/test_prompt_cache.py new file mode 100644 index 00000000..0793dda8 --- /dev/null +++ b/agentigrid/tests/llm/test_prompt_cache.py @@ -0,0 +1,229 @@ +"""Prompt caching (Anthropic) and per-run token metering. + +Caching must be billing-only: the model sees the same system text either way, +``prompt_tokens`` keeps meaning TOTAL input tokens, and the cache breakdown is +reported separately. The UsageMeter must count every call without altering it. +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from agentigrid.backends import LLMBackend, LLMResponse, UsageMeter +from agentigrid.config import LLMConfig, load_config + +try: + import anthropic as _anthropic # noqa: F401 + _has_anthropic = True +except ImportError: + _has_anthropic = False + + +def _cfg(**over) -> LLMConfig: + base = dict( + backend="anthropic", model="claude-sonnet-4-6", api_key_env="TEST_API_KEY", + openai_base_url=None, ollama_host="http://localhost:11434", + ollama_cloud_host=None, temperature=0.3, max_tokens=256, + ) + base.update(over) + return LLMConfig(**base) + + +def _fake_message(input_tokens, creation, read, output_tokens=7, text='{"action":"complete"}'): + usage = SimpleNamespace( + input_tokens=input_tokens, + cache_creation_input_tokens=creation, + cache_read_input_tokens=read, + output_tokens=output_tokens, + ) + return SimpleNamespace(content=[SimpleNamespace(type="text", text=text)], usage=usage) + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +def test_prompt_cache_defaults_on(): + cfg = load_config(None) + assert cfg.llm.prompt_cache is True + + +def test_prompt_cache_can_be_disabled_by_override(): + cfg = load_config(None, cli_overrides={"llm.prompt_cache": False}) + assert cfg.llm.prompt_cache is False + + +# --------------------------------------------------------------------------- +# Anthropic backend request shape + token accounting +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_system_prompt_sent_as_cached_block(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=True)) + with patch.object(backend._client.messages, "create", + return_value=_fake_message(50, 4000, 0)) as create: + backend.complete("SYSTEM TEXT", "user text") + system = create.call_args.kwargs["system"] + assert system == [{"type": "text", "text": "SYSTEM TEXT", + "cache_control": {"type": "ephemeral"}}] + # the user turn is untouched + assert create.call_args.kwargs["messages"] == [{"role": "user", "content": "user text"}] + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_cache_off_sends_plain_string(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=False)) + with patch.object(backend._client.messages, "create", + return_value=_fake_message(4050, None, None)) as create: + resp = backend.complete("SYSTEM TEXT", "user text") + assert create.call_args.kwargs["system"] == "SYSTEM TEXT" + assert resp.prompt_tokens == 4050 + assert resp.cache_creation_tokens is None and resp.cache_read_tokens is None + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic package not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_prompt_tokens_is_total_input_with_breakdown(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + + backend = AnthropicBackend(_cfg(prompt_cache=True)) + # first call writes the cache, second reads it + with patch.object(backend._client.messages, "create", + side_effect=[_fake_message(50, 4000, 0), _fake_message(60, 0, 4000)]): + first = backend.complete("S", "u1") + second = backend.complete("S", "u2") + assert (first.prompt_tokens, first.cache_creation_tokens, first.cache_read_tokens) == (4050, 4000, 0) + assert (second.prompt_tokens, second.cache_creation_tokens, second.cache_read_tokens) == (4060, 0, 4000) + assert first.json_data == {"action": "complete"} + + +# --------------------------------------------------------------------------- +# UsageMeter +# --------------------------------------------------------------------------- + +class _Stub(LLMBackend): + def __init__(self, responses): + self._responses = list(responses) + self.seen = [] + + def complete(self, system_prompt, user_prompt, temperature=None): + self.seen.append((system_prompt, user_prompt, temperature)) + return self._responses.pop(0) + + def name(self): + return "stub" + + def supports_json_mode(self): + return False + + def custom(self): + return "delegated" + + +def _resp(pt, ct, cw=None, cr=None): + return LLMResponse(raw_text="x", json_data=None, json_error=None, model="m", + backend="stub", prompt_tokens=pt, completion_tokens=ct, + cache_creation_tokens=cw, cache_read_tokens=cr) + + +def test_usage_meter_counts_all_calls_and_is_transparent(): + inner = _Stub([_resp(4050, 10, 4000, 0), _resp(4060, 12, 0, 4000), _resp(None, None)]) + meter = UsageMeter(inner) + r1 = meter.complete("S", "u1") + meter.complete("S", "u2", temperature=0.0) + meter.complete("S", "u3") + assert r1 is not None and r1.prompt_tokens == 4050 + assert inner.seen[1] == ("S", "u2", 0.0) # args passed through verbatim + assert meter.totals() == { + "calls": 3, "prompt_tokens": 8110, "completion_tokens": 22, + "cache_creation_tokens": 4000, "cache_read_tokens": 4000, + "api_errors": 0, "truncated": 0, "temperature_sent": None, + } + assert meter.name() == "stub" + assert meter.custom() == "delegated" # backend-specific attrs delegate + + +def test_journal_exports_llm_usage(tmp_path): + import json + from agentigrid.engine.journal import SearchJournal + + j = SearchJournal() + j.llm_usage = {"calls": 2, "prompt_tokens": 100, "completion_tokens": 5, + "cache_creation_tokens": 80, "cache_read_tokens": 0} + out = tmp_path / "j.json" + j.export_json(out) + assert json.loads(out.read_text())["llm_usage"]["cache_creation_tokens"] == 80 + + +def test_temperature_rejected_retries_without_it_and_remembers(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg(model="claude-sonnet-5")) + calls = [] + + def create(**kw): + calls.append(kw) + if "temperature" in kw: + raise RuntimeError("Error code: 400 - `temperature` is deprecated for this model.") + return _fake_message(10, 0, 0) + + with patch.object(backend._client.messages, "create", side_effect=create): + r1 = backend.complete("sys", "u1") + r2 = backend.complete("sys", "u2") + assert not r1.api_error and r1.json_data == {"action": "complete"} + assert ["temperature" in c for c in calls] == [True, False, False] # retried once, then omitted + assert backend.temperature_sent is False and r2.prompt_tokens == 10 + + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_other_api_errors_are_flagged_and_counted(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg()) + meter = UsageMeter(backend) + with patch.object(backend._client.messages, "create", side_effect=RuntimeError("401 invalid x-api-key")): + r = meter.complete("sys", "u") + assert r.api_error and r.json_data is None + t = meter.totals() + assert t["calls"] == 1 and t["api_errors"] == 1 and t["prompt_tokens"] == 0 + with patch.object(backend._client.messages, "create", return_value=_fake_message(5, 0, 0)): + meter.complete("sys", "u") + assert meter.totals()["temperature_sent"] is True + + +# --------------------------------------------------------------------------- +# Output-token cap: stop_reason telemetry and AGENTIGRID_MAX_TOKENS +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not _has_anthropic, reason="anthropic not installed") +@patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}) +def test_max_tokens_stop_is_reported_and_counted(): + from agentigrid.backends.anthropic_backend import AnthropicBackend + backend = AnthropicBackend(_cfg()) + meter = UsageMeter(backend) + msg = _fake_message(10, 0, 0, output_tokens=256, text="") + msg.content = [SimpleNamespace(type="thinking", thinking="...")] + msg.stop_reason = "max_tokens" + with patch.object(backend._client.messages, "create", return_value=msg): + r = meter.complete("sys", "u") + assert r.stop_reason == "max_tokens" and r.content_types == ["thinking"] + assert r.raw_text == "" and r.json_data is None and not r.api_error + assert meter.totals()["truncated"] == 1 + + +def test_max_tokens_env_override(monkeypatch): + monkeypatch.setenv("AGENTIGRID_MAX_TOKENS", "16000") + assert load_config(None).llm.max_tokens == 16000 + monkeypatch.setenv("AGENTIGRID_MAX_TOKENS", "lots") + assert load_config(None).llm.max_tokens == 4096 # ignored, default kept + monkeypatch.delenv("AGENTIGRID_MAX_TOKENS") + assert load_config(None).llm.max_tokens == 4096 diff --git a/agentigrid/tests/results/test_opflow_parser.py b/agentigrid/tests/results/test_opflow_parser.py index f0cedf0c..54a9b42f 100644 --- a/agentigrid/tests/results/test_opflow_parser.py +++ b/agentigrid/tests/results/test_opflow_parser.py @@ -51,10 +51,10 @@ def test_objective_type(self, result: OPFLOWResult): assert result.objective_type == "MIN_GEN_COST" def test_num_iterations(self, result: OPFLOWResult): - assert result.num_iterations == 23 + assert result.num_iterations == 21 def test_solve_time(self, result: OPFLOWResult): - assert result.solve_time == pytest.approx(0.042, abs=0.001) + assert result.solve_time == pytest.approx(0.021, abs=0.001) def test_bus_count(self, result: OPFLOWResult): assert len(result.buses) == 200 @@ -91,10 +91,18 @@ def test_branch_187_189_highest_sf(self, result: OPFLOWResult): assert abs(max_br.Sf - 387.19) < 0.01 def test_gen_189(self, result: OPFLOWResult): - """Gen at bus 189: Pg≈383.40, fuel=COAL.""" + """Gen at bus 189: Pg≈383.40, fuel=NUCLEAR (mpc.genfuel of case_ACTIVSg200.m).""" gen189 = next(g for g in result.generators if g.bus == 189) assert abs(gen189.Pg - 383.40) < 0.01 - assert gen189.fuel == "COAL" + assert gen189.fuel == "NUCLEAR" + + def test_fuel_mix_matches_case_file(self, result: OPFLOWResult): + """All 49 units keep the fuel from mpc.genfuel (25 coal, 17 ng, 6 wind, + 1 nuclear). A fixture regenerated with a writer that drops the "};" + after mpc.genfuel would show every unit as COAL.""" + from collections import Counter + assert Counter(g.fuel for g in result.generators) == { + "COAL": 25, "NG": 17, "WIND": 6, "NUCLEAR": 1} # =========================================================================== diff --git a/agentigrid/tests/runner/test_executor.py b/agentigrid/tests/runner/test_executor.py index 1b46cbd5..3c89c92b 100644 --- a/agentigrid/tests/runner/test_executor.py +++ b/agentigrid/tests/runner/test_executor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import shutil from pathlib import Path from unittest.mock import patch, MagicMock @@ -331,6 +332,8 @@ def test_no_mpirun_for_sopflow_with_mpi_np_1(self, tmp_path: Path): assert cmd_arg[0] != "mpirun" +@pytest.mark.skipif(not _has_opflow, reason="opflow binary not available") +@pytest.mark.skipif(not _has_test_file, reason="case_ACTIVSg200.m not in data/exago/examples/") class TestExecutorLive: def test_opflow_run(self, tmp_path: Path): @@ -350,8 +353,11 @@ def test_opflow_run(self, tmp_path: Path): assert result.elapsed_seconds < exago.timeout assert result.exit_code == 0 or result.success # MPI noise tolerance - # Save stdout for Step 1.6 (only if converged, to avoid overwriting good sample) - if "Optimal Solution Found" in result.stdout: + # Regenerate the parser fixture only on request: the parser tests in + # tests/results/ hard-code values from it, so overwriting it on every + # live run made their results depend on the machine and test order. + # AGENTIGRID_REGENERATE_FIXTURES=1 python -m pytest tests/runner/test_executor.py -k test_opflow_run + if os.environ.get("AGENTIGRID_REGENERATE_FIXTURES") == "1" and "Optimal Solution Found" in result.stdout: sample_path = Path(__file__).resolve().parent.parent / "fixtures" / "sample_opflow_output.txt" sample_path.write_text(result.stdout, encoding="utf-8") print(f"\nSaved sample output to {sample_path} ({len(result.stdout)} bytes)")