AgentiGrid: robust agent iterations, accurate UI timeline, LLM usage accounting

agentigrid/README.md

@@ -367,7 +367,7 @@ agentigrid ./data/exago/examples/case_ACTIVSg200.m "Find the maximum load scalin
 
 # With options
 agentigrid ./data/exago/examples/case_ACTIVSg200.m "Minimize generation cost" \
-  --backend anthropic --model claude-sonnet-4-20250514 \
+  --backend anthropic --model claude-sonnet-4-6 \
   --app opflow --max-iter 30 --verbose
 
 # Quiet mode (only show final summary)
@@ -424,7 +424,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r
   Version 0.1.0
 ============================================================
   Backend:        anthropic
-  Model:          claude-sonnet-4-20250514
+  Model:          claude-sonnet-4-6
   Application:    opflow
   Base case:      data/exago/examples/case_ACTIVSg200.m
   Goal:           Find the maximum load scaling factor
@@ -435,7 +435,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r
 [Iter 0] Running base case simulation...
 [Iter 0] Base case: CONVERGED, cost=$27,557.57
 
-[Iter 1] Sending prompt to anthropic (claude-sonnet-4-20250514)...
+[Iter 1] Sending prompt to anthropic (claude-sonnet-4-6)...
 [Iter 1] LLM action: modify — "Scale all loads +20%"
 [Iter 1] Applied 1 command(s), 0 skipped
 [Iter 1] Simulation completed in 0.04s — CONVERGED, cost=$33,019.55
@@ -450,7 +450,7 @@ python -m agentigrid ./data/exago/examples/case_ACTIVSg200.m "test goal" --dry-r
 ============================================================
   Goal:           Find the maximum load scaling factor
   Application:    opflow
-  Backend:        anthropic (claude-sonnet-4-20250514)
+  Backend:        anthropic (claude-sonnet-4-6)
   Iterations:     6 (of max 20)
   Duration:       18.3 seconds
   Tokens used:    ~12,450 (prompt: 9,200, completion: 3,250)

agentigrid/agentigrid/backends/__init__.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
-from agentigrid.backends.base import LLMBackend, LLMResponse
+from agentigrid.backends.base import LLMBackend, LLMResponse, UsageMeter
 from agentigrid.config import LLMConfig
 
-__all__ = ["LLMBackend", "LLMResponse", "create_backend"]
+__all__ = ["LLMBackend", "LLMResponse", "UsageMeter", "create_backend"]
 
 
 def create_backend(config: LLMConfig) -> LLMBackend:

agentigrid/agentigrid/backends/anthropic_backend.py

@@ -27,6 +27,9 @@ def __init__(self, config: LLMConfig) -> None:
             raise ImportError("anthropic package is required for the Anthropic backend")
 
         self._config = config
+        # None = not yet known; True/False once a call has succeeded / the
+        # model has rejected the parameter (see complete()).
+        self.temperature_sent: Optional[bool] = None
         api_key = os.environ.get(config.api_key_env)
         if not api_key:
             logger.warning(
@@ -40,6 +43,26 @@ def name(self) -> str:
         """Return the backend name."""
         return "anthropic"
 
+    def _system_param(self, system_prompt: str):
+        """System prompt in the form the Messages API expects.
+
+        With ``prompt_cache`` on, the system prompt is sent as a single text block
+        carrying an ephemeral ``cache_control`` breakpoint. AgentiGrid builds the
+        system prompt once per session, so every later iteration reads it from the
+        cache instead of paying the full input price again. This affects billing
+        only; the model sees identical text either way. Prompts below the model's
+        minimum cacheable length are silently processed uncached by the API.
+        """
+        if not getattr(self._config, "prompt_cache", False):
+            return system_prompt
+        return [
+            {
+                "type": "text",
+                "text": system_prompt,
+                "cache_control": {"type": "ephemeral"},
+            }
+        ]
+
     def supports_json_mode(self) -> bool:
         """Anthropic does not have a native JSON output mode."""
         return False
@@ -54,20 +77,45 @@ def complete(
         temp = temperature if temperature is not None else self._config.temperature
 
         try:
-            response = self._client.messages.create(
+            kwargs = dict(
                 model=self._config.model,
                 max_tokens=self._config.max_tokens,
-                temperature=temp,
-                system=system_prompt,
+                system=self._system_param(system_prompt),
                 messages=[{"role": "user", "content": user_prompt}],
             )
+            if self.temperature_sent is not False:
+                kwargs["temperature"] = temp
+            try:
+                response = self._client.messages.create(**kwargs)
+                if "temperature" in kwargs:
+                    self.temperature_sent = True
+            except Exception as exc:
+                # Newer models reject the sampling-temperature parameter
+                # ("`temperature` is deprecated for this model"). Retry once
+                # without it and stop sending it for the rest of the session;
+                # the journal records temperature_sent=False.
+                if "temperature" in kwargs and "temperature" in str(exc).lower():
+                    logger.warning("Model %s rejects `temperature`; retrying without it.", self._config.model)
+                    kwargs.pop("temperature")
+                    self.temperature_sent = False
+                    response = self._client.messages.create(**kwargs)
+                else:
+                    raise
 
             raw_text = ""
+            content_types = []
             for block in response.content:
+                content_types.append(getattr(block, "type", None))
                 if block.type == "text":
                     raw_text += block.text
+            stop_reason = getattr(response, "stop_reason", None)
+            if stop_reason == "max_tokens":
+                logger.warning("Response stopped at max_tokens=%s (blocks: %s, %d text chars)",
+                               self._config.max_tokens, content_types, len(raw_text))
 
-            prompt_tokens = getattr(response.usage, "input_tokens", None)
+            prompt_tokens, cache_creation, cache_read = _input_token_breakdown(
+                response.usage
+            )
             completion_tokens = getattr(response.usage, "output_tokens", None)
 
             json_data, json_error = extract_json(raw_text)
@@ -80,6 +128,10 @@ def complete(
                 backend=self.name(),
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
+                cache_creation_tokens=cache_creation,
+                cache_read_tokens=cache_read,
+                stop_reason=stop_reason,
+                content_types=content_types,
             )
 
         except Exception as exc:
@@ -93,4 +145,23 @@ def complete(
                 backend=self.name(),
                 prompt_tokens=None,
                 completion_tokens=None,
+                api_error=True,
             )
+
+
+def _input_token_breakdown(usage) -> tuple[Optional[int], Optional[int], Optional[int]]:
+    """Return (total_input, cache_creation, cache_read) from an API usage object.
+
+    With caching, the API's ``input_tokens`` counts only tokens after the last
+    cache breakpoint. The total input is ``input_tokens + cache_creation_input_tokens
+    + cache_read_input_tokens``; returning that total keeps ``prompt_tokens``
+    comparable with runs made before caching was enabled.
+    """
+    base = getattr(usage, "input_tokens", None)
+    creation = getattr(usage, "cache_creation_input_tokens", None)
+    read = getattr(usage, "cache_read_input_tokens", None)
+    creation = creation if isinstance(creation, int) else None
+    read = read if isinstance(read, int) else None
+    if not isinstance(base, int):
+        return None, creation, read
+    return base + (creation or 0) + (read or 0), creation, read

agentigrid/agentigrid/backends/base.py

@@ -18,6 +18,20 @@ class LLMResponse:
     backend: str
     prompt_tokens: Optional[int]
     completion_tokens: Optional[int]
+    # Prompt-cache accounting (Anthropic). ``prompt_tokens`` always reports the
+    # TOTAL input tokens (uncached + cache writes + cache reads), so its meaning
+    # is the same with or without caching; these break that total down.
+    cache_creation_tokens: Optional[int] = None
+    cache_read_tokens: Optional[int] = None
+    # True when the request itself failed (network, auth, invalid request...),
+    # as opposed to the model answering with unparseable text. Lets the agent
+    # loop record API failures separately from rejected proposals.
+    api_error: bool = False
+    # Why generation stopped ("end_turn", "max_tokens", ...) and the content
+    # block types returned, when the backend reports them. A "max_tokens" stop
+    # is an output-cap limit, not a model proposal, and is recorded as such.
+    stop_reason: Optional[str] = None
+    content_types: Optional[list] = None
 
 
 class LLMBackend(ABC):
@@ -51,3 +65,61 @@ def name(self) -> str:
     def supports_json_mode(self) -> bool:
         """Whether this backend supports a native JSON output mode."""
         ...
+
+
+class UsageMeter(LLMBackend):
+    """Transparent wrapper that meters token usage across EVERY backend call.
+
+    The agent loop's own counters cover only the main per-iteration call; the
+    post-search analysis calls are not counted there. Wrapping the backend once
+    gives complete per-run totals (including prompt-cache writes and reads) for
+    cost accounting, without touching any call site. Behaviour is unchanged:
+    every call is delegated verbatim.
+    """
+
+    def __init__(self, inner: LLMBackend) -> None:
+        self._inner = inner
+        self.calls = 0
+        self.prompt_tokens = 0
+        self.completion_tokens = 0
+        self.cache_creation_tokens = 0
+        self.cache_read_tokens = 0
+        self.api_errors = 0
+        self.truncated = 0
+
+    def complete(self, system_prompt, user_prompt, temperature=None):
+        resp = self._inner.complete(system_prompt, user_prompt, temperature)
+        self.calls += 1
+        if getattr(resp, "api_error", False):
+            self.api_errors += 1
+        if getattr(resp, "stop_reason", None) == "max_tokens":
+            self.truncated += 1
+        self.prompt_tokens += resp.prompt_tokens or 0
+        self.completion_tokens += resp.completion_tokens or 0
+        self.cache_creation_tokens += resp.cache_creation_tokens or 0
+        self.cache_read_tokens += resp.cache_read_tokens or 0
+        return resp
+
+    def name(self) -> str:
+        return self._inner.name()
+
+    def supports_json_mode(self) -> bool:
+        return self._inner.supports_json_mode()
+
+    def __getattr__(self, item):  # delegate anything backend-specific
+        if item == "_inner":  # not yet set (e.g. during copy/unpickle)
+            raise AttributeError(item)
+        return getattr(self._inner, item)
+
+    def totals(self) -> dict:
+        """Per-run usage totals, as written to the journal (``llm_usage``)."""
+        return {
+            "calls": self.calls,
+            "prompt_tokens": self.prompt_tokens,
+            "completion_tokens": self.completion_tokens,
+            "cache_creation_tokens": self.cache_creation_tokens,
+            "cache_read_tokens": self.cache_read_tokens,
+            "api_errors": self.api_errors,
+            "truncated": self.truncated,
+            "temperature_sent": getattr(self._inner, "temperature_sent", None),
+        }

agentigrid/agentigrid/backends/ollama_backend.py

@@ -120,4 +120,5 @@ def complete(
                 backend=self.name(),
                 prompt_tokens=None,
                 completion_tokens=None,
+                api_error=True,
             )

agentigrid/agentigrid/backends/openai_backend.py

@@ -104,4 +104,5 @@ def complete(
                 backend=self.name(),
                 prompt_tokens=None,
                 completion_tokens=None,
+                api_error=True,
             )

agentigrid/agentigrid/config.py

@@ -36,13 +36,14 @@
     },
     "llm": {
         "backend": "anthropic",
-        "model": "claude-sonnet-4-20250514",
+        "model": "claude-sonnet-4-6",
         "api_key_env": "ANTHROPIC_API_KEY",
         "openai_base_url": None,
         "ollama_host": "http://localhost:11434",
         "ollama_cloud_host": None,
         "temperature": 0.3,
         "max_tokens": 4096,
+        "prompt_cache": True,
     },
     "search": {
         "max_iterations": 20,
@@ -159,6 +160,9 @@ class LLMConfig:
     ollama_cloud_host: Optional[str]
     temperature: float
     max_tokens: int
+    # Anthropic prompt caching of the (per-session constant) system prompt.
+    # Billing-only: does not change model outputs. Ignored by other backends.
+    prompt_cache: bool = True
 
 
 @dataclass(frozen=True)
@@ -374,6 +378,15 @@ def load_config(
             target[parts[-1]] = value
         logger.debug("Applied CLI overrides: %s", cli_overrides)
 
+    # AGENTIGRID_MAX_TOKENS overrides the output-token cap (e.g. to fix it for a
+    # batch of runs); the value used is recorded in the journal's llm_usage.
+    _mt = os.environ.get("AGENTIGRID_MAX_TOKENS", "").strip()
+    if _mt:
+        try:
+            merged["llm"] = {**merged["llm"], "max_tokens": int(_mt)}
+        except ValueError:
+            logger.warning("Ignoring non-integer AGENTIGRID_MAX_TOKENS=%r", _mt)
+
     # Build frozen dataclass sections
     exago_path_fields = {
         "binary_dir", "opflow_binary", "scopflow_binary", "tcopflow_binary",