diff --git a/agentigrid/.gitignore b/agentigrid/.gitignore index 61a7b70f..9b0aad9e 100644 --- a/agentigrid/.gitignore +++ b/agentigrid/.gitignore @@ -114,3 +114,7 @@ desktop.ini AGENTS.md opencode.md /prompts/ + +# RAG index (build locally: python -m agentigrid.rag.ingest rag/corpus) +rag/store/ +rag/store_*/ diff --git a/agentigrid/README.md b/agentigrid/README.md index 59f6fdda..085af19a 100644 --- a/agentigrid/README.md +++ b/agentigrid/README.md @@ -286,6 +286,10 @@ When resuming, the goal and journal are loaded from the saved session, but the L The launcher sidebar includes a "Session Save/Resume" section with a Save button (available after search completes or while running) and a dropdown to resume from previously saved sessions. +## Retrieval Grounding (optional RAG) + +AgentiGrid can ground the LLM's proposals in a curated, provenance-tagged corpus (`rag/corpus/`: ExaGO tool help, case metadata, certified response examples). It is **off by default**, touches only the generation step (never the deterministic validator), and degrades to the no-RAG baseline on any retrieval failure. Modes are `off`, `basic` and `corrective`, set with `AGENTIGRID_RAG_MODE` or the UI's **Advanced** panel. Setup, optional dependencies (`requirements-rag.txt`) and troubleshooting: [docs/RAG_USAGE.md](docs/RAG_USAGE.md). + ## Interactive Steering While a search is running, you can inject steering directives from the terminal (CLI) or the GUI — without stopping and restarting the search. diff --git a/agentigrid/agentigrid/exago_engine/agent_loop.py b/agentigrid/agentigrid/exago_engine/agent_loop.py index 10165cce..13de8259 100644 --- a/agentigrid/agentigrid/exago_engine/agent_loop.py +++ b/agentigrid/agentigrid/exago_engine/agent_loop.py @@ -47,6 +47,7 @@ results_summary_for_app, ) from agentigrid.exago_parsers.matpower_model import MATNetwork +from agentigrid.rag import build_retriever from agentigrid.exago_parsers.opflow_results import OPFLOWResult from agentigrid.exago_engine import topology from agentigrid.exago_engine import contingency @@ -771,6 +772,13 @@ def __init__( # Metered so per-run token totals (incl. prompt-cache reads/writes) # cover every LLM call, not only the main per-iteration call. self._backend: LLMBackend = UsageMeter(create_backend(config.llm)) + # Optional retrieval grounding (off by default). AGENTIGRID_RAG_MODE is one + # of off | basic | corrective (AGENTIGRID_RAG=1/0 still maps to basic/off). + # Every mode exposes the same .enabled / .retrieve() surface, and "off" + # needs none of the optional RAG dependencies. + self._retriever = build_retriever( + host=os.environ.get("OLLAMA_HOST") or getattr(config.llm, "ollama_host", None) or "http://localhost:11434", + ) self._executor = SimulationExecutor(config.exago, config.output) self._journal = SearchJournal() self._quiet = quiet @@ -1457,7 +1465,22 @@ def _iteration( ) self._error_feedback = None # consumed + # Retrieval grounds the generation step only; the deterministic validator + # never sees it. A failed or empty retrieval leaves the prompt unchanged. + _retrieved = self._retriever.retrieve(goal) if self._retriever.enabled else "" + if _retrieved: + user_prompt = ( + "=== Section B: Reference Material (retrieved) ===\n" + f"{_retrieved}\n\n{user_prompt}" + ) + if self._on_phase: + if self._retriever.enabled: + # "rag_retrieved::" feeds the UI's grounding caption + # (0 refs under corrective = context withheld). + _scores = [float(x) for x in re.findall(r"score (\d+\.\d+)", _retrieved)] + _top = max(_scores) if _scores else 0.0 + self._on_phase(iteration, f"rag_retrieved:{_retrieved.count('[ref ')}:{_top:.2f}") self._on_phase(iteration, "llm_request") response = self._backend.complete(system_prompt, user_prompt) @@ -4999,6 +5022,19 @@ def _assemble_prompt( def _finalize(self, session: SearchSession, elapsed_seconds: float) -> None: """Print summary and save journal.""" + _rag_enabled = self._retriever.enabled + self._journal.rag_enabled = _rag_enabled + try: + from agentigrid.rag import resolve_rag_mode, resolve_crag_grader + _desc = getattr(self._retriever, "describe", None) + self._journal.rag_config = { + **(_desc() if callable(_desc) else {}), + "mode_env": resolve_rag_mode(), + "grader_env": resolve_crag_grader(), + } + except Exception: + self._journal.rag_config = None + session.rag_enabled = _rag_enabled _totals = getattr(self._backend, "totals", None) if callable(_totals): self._journal.llm_usage = { diff --git a/agentigrid/agentigrid/exago_engine/journal.py b/agentigrid/agentigrid/exago_engine/journal.py index 0c7eea3f..6d0e556f 100644 --- a/agentigrid/agentigrid/exago_engine/journal.py +++ b/agentigrid/agentigrid/exago_engine/journal.py @@ -1131,8 +1131,12 @@ def export_json(self, path: Path) -> None: data["session_best"] = self.session_best if self.load_factor is not None: data["load_factor"] = self.load_factor + if getattr(self, "rag_enabled", None) is not None: + data["rag_enabled"] = self.rag_enabled if getattr(self, "llm_usage", None) is not None: data["llm_usage"] = self.llm_usage + if getattr(self, "rag_config", None) is not None: + data["rag_config"] = self.rag_config if getattr(self, "discarded_actions", None): data["discarded_actions"] = self.discarded_actions path.write_text(json.dumps(data, indent=2), encoding="utf-8") diff --git a/agentigrid/agentigrid/rag/__init__.py b/agentigrid/agentigrid/rag/__init__.py new file mode 100755 index 00000000..b0fba5f2 --- /dev/null +++ b/agentigrid/agentigrid/rag/__init__.py @@ -0,0 +1,108 @@ +"""RAG (retrieval-augmented generation) support for AgentiGrid. + +Generation-stage only: retrieval grounds the LLM's spec/proposal generation. +It is NEVER used by the deterministic validator (engine/validation.py). + +Retrieval is OFF by default. The mode is selected by an env var: + + AGENTIGRID_RAG_MODE = off | basic | corrective + +For backward compatibility the legacy switch still works: + AGENTIGRID_RAG=1 -> basic + AGENTIGRID_RAG=0/unset (and no _MODE) -> off + +Corrective-mode grader (AGENTIGRID_CRAG_GRADER): cosine (default) | reranker +(local cross-encoder, needs sentence-transformers). Thresholds: +AGENTIGRID_CRAG_TAU_LOWER / AGENTIGRID_CRAG_TAU_UPPER. + +Use `build_retriever(**kwargs)` to get the retriever for the active mode; every +mode exposes the same surface (`.enabled` and `.retrieve(query) -> str`). +""" +from __future__ import annotations + +import os + +from .retriever import Retriever +from .corrective import CorrectiveRetriever + +VALID_MODES = ("off", "basic", "corrective") +# Graders implemented behind the corrective-RAG seam. +VALID_CRAG_GRADERS = ("cosine", "reranker") + + +def resolve_rag_mode() -> str: + """Return the active RAG mode, honoring AGENTIGRID_RAG_MODE and falling back + to the legacy AGENTIGRID_RAG switch. Unknown values degrade to 'basic'. + """ + m = os.environ.get("AGENTIGRID_RAG_MODE") + if m: + m = m.strip().lower() + return m if m in VALID_MODES else "basic" + return "basic" if os.environ.get("AGENTIGRID_RAG", "0") == "1" else "off" + + +def resolve_crag_grader() -> str: + """Return the corrective-RAG grader name from AGENTIGRID_CRAG_GRADER. + 'cosine' (default/unset) or 'reranker'. Unknown values fall back to 'cosine'. + """ + name = os.environ.get("AGENTIGRID_CRAG_GRADER", "").strip().lower() + return name if name in VALID_CRAG_GRADERS else "cosine" + + +def _make_crag_grader(): + """Build the optional grader for corrective mode (selected by + AGENTIGRID_CRAG_GRADER). 'cosine' -> None (deterministic cosine buckets, the + reproducible default). 'reranker' -> RerankerGrader, imported lazily so + sentence-transformers is only needed when it is actually requested. + """ + name = resolve_crag_grader() + if name == "reranker": + from .grader_reranker import RerankerGrader + return RerankerGrader() # loads lazily; failures are counted, not hidden + return None # 'cosine' (and any unknown value already normalized) + + +def _tau_overrides() -> dict: + """Corrective thresholds from AGENTIGRID_CRAG_TAU_LOWER / _UPPER, if set. + + Each grader's score scale differs (cosine vs. reranker score vs. calibrated + probability), so its thresholds are fixed on labeled data and passed per + run; unset = the module defaults (0.30 / 0.50, cosine). + """ + out = {} + for env, key in (("AGENTIGRID_CRAG_TAU_LOWER", "tau_lower"), ("AGENTIGRID_CRAG_TAU_UPPER", "tau_upper")): + val = os.environ.get(env) + if val not in (None, ""): + out[key] = float(val) + if "tau_lower" in out and "tau_upper" in out and out["tau_lower"] > out["tau_upper"]: + raise ValueError("AGENTIGRID_CRAG_TAU_LOWER must not exceed AGENTIGRID_CRAG_TAU_UPPER") + return out + + +def build_retriever(**base_kwargs): + """Construct the retriever for the active mode. + + `base_kwargs` are passed to the basic Retriever (e.g. host=, path=, model=). + Returns a Retriever (off/basic) or a CorrectiveRetriever (corrective); both + share the `.enabled` / `.retrieve()` surface. + """ + mode = resolve_rag_mode() + base_kwargs.pop("enabled", None) + # Corpus selection (e.g. core vs. a larger documentation corpus): each + # corpus is ingested into its own store directory. + if os.environ.get("AGENTIGRID_RAG_STORE"): + base_kwargs["path"] = os.environ["AGENTIGRID_RAG_STORE"] + if os.environ.get("AGENTIGRID_RAG_COLLECTION"): + base_kwargs["collection"] = os.environ["AGENTIGRID_RAG_COLLECTION"] + if mode == "off": + return Retriever(enabled=False) + base = Retriever(enabled=True, **base_kwargs) + if mode == "corrective": + return CorrectiveRetriever(base, grader=_make_crag_grader(), **_tau_overrides()) + return base # 'basic' (and any unknown value already normalized to basic) + + +__all__ = [ + "Retriever", "CorrectiveRetriever", "build_retriever", + "resolve_rag_mode", "resolve_crag_grader", +] diff --git a/agentigrid/agentigrid/rag/corpus_hash.py b/agentigrid/agentigrid/rag/corpus_hash.py new file mode 100644 index 00000000..0e3e2749 --- /dev/null +++ b/agentigrid/agentigrid/rag/corpus_hash.py @@ -0,0 +1,73 @@ +"""Corpus identity: which files form a corpus and its SHA-256. + +Shared by rag/tools/corpus_guard.py (freeze), agentigrid.rag.ingest (records the +hash of what it indexed) and corpus_status() (checks that a store was built +from the frozen corpus it claims). One definition, so they can never disagree. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +CORPUS_SUFFIXES = (".txt", ".md") # what ingest indexes +MANIFEST_NAME = "corpus_manifest.json" # written by corpus_guard --freeze +INGEST_MANIFEST_NAME = "ingest_manifest.json" # written by ingest into the store dir + +# How ingest splits the corpus into chunks. Part of the store's identity: the same +# corpus chunked differently retrieves different text, so corpus_status refuses a +# store built with another scheme. v2 keeps every paragraph (one worked example, +# one schema exemplar) whole up to CHUNK_SIZE characters; v1 (800/100) cut most +# worked examples mid-JSON. +CHUNKING = {"scheme": "paragraph-v2", "size": 3000, "overlap": 200} + + +def corpus_files(corpus_dir: Path) -> list[Path]: + corpus_dir = Path(corpus_dir) + return sorted(p for p in corpus_dir.rglob("*") + if p.is_file() and p.suffix in CORPUS_SUFFIXES and p.name != MANIFEST_NAME) + + +def corpus_hash(corpus_dir: Path) -> tuple[str, dict[str, str]]: + """Overall SHA-256 over (relative path, content hash) pairs, order-independent of disk.""" + corpus_dir = Path(corpus_dir) + per_file = {} + for p in corpus_files(corpus_dir): + per_file[p.relative_to(corpus_dir).as_posix()] = hashlib.sha256(p.read_bytes()).hexdigest() + h = hashlib.sha256() + for rel in sorted(per_file): + h.update(f"{rel}\0{per_file[rel]}\n".encode()) + return h.hexdigest(), per_file + + +def corpus_status(corpus_dir: Path, store_dir: Path) -> dict: + """Is *corpus_dir* frozen, unchanged since freezing, and is *store_dir* built + from exactly that corpus? Returns {"ok": bool, "reason": str|None, "corpus_sha256": ...}.""" + corpus_dir, store_dir = Path(corpus_dir), Path(store_dir) + out = {"ok": False, "reason": None, "corpus_dir": str(corpus_dir), "store_dir": str(store_dir), + "corpus_sha256": None} + mpath = corpus_dir / MANIFEST_NAME + if not mpath.exists(): + out["reason"] = f"corpus not frozen ({mpath} missing; run corpus_guard.py --freeze)" + return out + frozen = json.loads(mpath.read_text()).get("corpus_sha256") + digest, _ = corpus_hash(corpus_dir) + out["corpus_sha256"] = digest + if digest != frozen: + out["reason"] = "corpus changed since it was frozen (re-run corpus_guard.py --freeze, then ingest)" + return out + ipath = store_dir / INGEST_MANIFEST_NAME + if not ipath.exists(): + out["reason"] = f"store not built by the current ingest ({ipath} missing; re-run ingest)" + return out + ing = json.loads(ipath.read_text()) + if ing.get("corpus_sha256") != digest: + out["reason"] = "store was built from a different corpus version (re-run ingest)" + return out + built = ing.get("chunking") or {} + if {k: built.get(k) for k in CHUNKING} != CHUNKING: + out["reason"] = (f"store was built with different chunking ({built or 'v1, 800/100'}; " + f"current {CHUNKING}); re-run ingest") + return out + out["ok"] = True + return out diff --git a/agentigrid/agentigrid/rag/corrective.py b/agentigrid/agentigrid/rag/corrective.py new file mode 100755 index 00000000..a09451e9 --- /dev/null +++ b/agentigrid/agentigrid/rag/corrective.py @@ -0,0 +1,248 @@ +"""Corrective RAG (CRAG) retriever — generation-stage only. + +Extends basic retrieval with a CRAG-style retrieval evaluator + knowledge +refinement, adapted to an OFFLINE OPF setting (there is no web-search fallback): + + 1. Retrieve candidate chunks (reuses the basic Retriever's vector store). + 2. Grade them -> CORRECT / AMBIGUOUS / INCORRECT. + The default grader is DETERMINISTIC (cosine-score buckets), so runs are + reproducible and it works even when the base LLM is too weak to grade. + The grader is a single swappable seam (the `grader=` callable) — an + LLM-based evaluator can be dropped in later WITHOUT touching anything else + here; it only has to return one relevance score per hit. + 3. Corrective action: + CORRECT -> refined internal knowledge only. + AMBIGUOUS -> refined internal knowledge + a hedge instruction. + INCORRECT -> reformulate the query once and re-retrieve; if it is still + INCORRECT, withhold context entirely (degrade to the no-RAG + baseline) rather than ground the model on misleading refs. + 4. Knowledge refinement (the CRAG decompose-recompose step): split kept chunks + into strips, re-score each strip against the query, drop weak strips, and + recompose the survivors — so only the relevant sentences reach the prompt. + +Like the basic retriever it is DEFENSIVE: any failure returns "" so a run +behaves exactly like the no-RAG baseline. It NEVER touches the deterministic +validator (engine/validation.py) — retrieval grounds generation only. + +Thresholds below are initial defaults, meant to be tuned on the real corpus; +they are not empirically calibrated yet. +""" +from __future__ import annotations + +import math +import re +from typing import Callable, Optional + +from .retriever import Retriever + +Hit = tuple[str, dict, float] + +CORRECT, AMBIGUOUS, INCORRECT = "correct", "ambiguous", "incorrect" + + +def _cosine(a: list[float], b: list[float]) -> float: + if not a or not b: + return 0.0 + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + if na == 0.0 or nb == 0.0: + return 0.0 + return dot / (na * nb) + + +def _strips(text: str) -> list[str]: + """Sentence/line-level decomposition; keep only non-trivial strips.""" + parts = re.split(r"(?<=[.!?])\s+|\n+", text) + return [p.strip() for p in parts if len(p.strip()) >= 20] + + +# Deterministic query reformulation: append domain keywords implied by the query. +# This is the offline stand-in for CRAG's web-search fallback. +_EXPANSION = { + "cost": "generation dispatch redispatch economic", + "load": "load scaling demand feasibility voltage limits", + "infeasible": "feasibility voltage limits thermal constraints", + "voltage": "voltage limits bus vlimits reactive power", + "outage": "N-1 contingency security scopflow", + "contingency": "N-1 contingency security scopflow", + "thermal": "branch rate line loading thermal limits", +} + + +class CorrectiveRetriever: + """CRAG retriever. Same public surface as ``Retriever``: ``.enabled`` and + ``.retrieve(query) -> str``, so the engine and UI use it interchangeably. + """ + + def __init__( + self, + base: Retriever, + *, + tau_lower: float = 0.30, + tau_upper: float = 0.50, + strip_min_score: float = 0.30, + max_refine_docs: int = 3, + max_strips_per_doc: int = 4, + grader: Optional[Callable[[str, list[Hit]], list[float]]] = None, + ): + self._base = base + self.enabled = base.enabled + self.k = base.k + self.tau_lower = tau_lower + self.tau_upper = tau_upper + self.strip_min_score = strip_min_score + self.max_refine_docs = max_refine_docs + self.max_strips_per_doc = max_strips_per_doc + # === Extension seam for an LLM grader === + # Provide a callable (query, hits) -> [score per hit in 0..1]; it replaces + # the cosine scores in _grade(). Everything else stays identical. + self._grader = grader + # Per-session telemetry (journaled via describe() as rag_config): lets the + # analysis see what corrective retrieval actually did in each run. + self.stats = {"calls": 0, "correct": 0, "ambiguous": 0, "incorrect": 0, + "rewrites": 0, "withheld": 0, "grader_fallbacks": 0} + + def describe(self) -> dict: + """Effective configuration for this run (shown in the UI / logged).""" + return { + "mode": "corrective", + "enabled": self.enabled, + "k": self.k, + "min_score": getattr(self._base, "min_score", None), + "store_path": getattr(self._base, "store_path", None), + "collection": getattr(self._base, "collection", None), + "tau_lower": self.tau_lower, + "tau_upper": self.tau_upper, + "strip_min_score": self.strip_min_score, + "grader": self._grader_name(), + "grader_detail": self._grader_detail(), + "stats": {**dict(self.stats), + "retrieval_errors": getattr(self._base, "stats", {}).get("errors", 0)}, + } + + def _grader_detail(self) -> Optional[dict]: + describe = getattr(self._grader, "describe", None) + if callable(describe): + try: + return describe() + except Exception: + return None + return None + + def _grader_name(self) -> str: + """Name of the active grader: 'cosine' when none is injected, otherwise + the grader's own reported name (e.g. 'reranker'), or its class name.""" + if self._grader is None: + return "cosine" + describe = getattr(self._grader, "describe", None) + if callable(describe): + try: + return describe().get("grader", type(self._grader).__name__) + except Exception: + return type(self._grader).__name__ + return getattr(self._grader, "__name__", type(self._grader).__name__) + + # -- grading ------------------------------------------------------- + def _scores(self, query: str, hits: list[Hit]) -> list[float]: + if self._grader is not None: + try: + return list(self._grader(query, hits)) + except Exception: + # any grader failure -> fall back to the cosine scores, COUNTED so + # the run is visibly not a clean run of the requested grader + self.stats["grader_fallbacks"] += 1 + return [s for (_d, _m, s) in hits] + + def _grade(self, scores: list[float]) -> str: + top = max(scores) if scores else 0.0 + if top >= self.tau_upper: + verdict = CORRECT + elif top < self.tau_lower: + verdict = INCORRECT + else: + verdict = AMBIGUOUS + self.stats[verdict] += 1 + return verdict + + # -- refinement (decompose-recompose) ------------------------------ + def _refine(self, query: str, hits: list[Hit]) -> list[tuple[str, float]]: + try: + q_emb = self._base.embed_text(query) + except Exception: + # can't embed -> fall back to whole surviving chunks + return [(d, s) for (d, _m, s) in hits] + out: list[tuple[str, float]] = [] + for (doc, _m, _s) in hits[: self.max_refine_docs]: + strips = _strips(doc)[: self.max_strips_per_doc] or [doc] + for strip in strips: + try: + sc = _cosine(q_emb, self._base.embed_text(strip)) + except Exception: + continue + if sc >= self.strip_min_score: + out.append((strip, sc)) + out.sort(key=lambda t: t[1], reverse=True) + return out + + # -- query reformulation (offline fallback) ------------------------ + def _reformulate(self, query: str) -> str: + ql = query.lower() + adds: list[str] = [] + for key, exp in _EXPANSION.items(): + if key in ql: + adds.extend(exp.split()) + if not adds: + return query + # de-duplicate while preserving order + return query + " " + " ".join(dict.fromkeys(adds)) + + # -- formatting ---------------------------------------------------- + def _format(self, refined: list[tuple[str, float]], verdict: str) -> str: + lines = [ + f"[ref {i + 1} | score {sc:.2f}] {txt}" + for i, (txt, sc) in enumerate(refined) + ] + header = ( + "Use the following retrieved reference material only when relevant; " + "do not invent facts beyond it." + ) + if verdict == AMBIGUOUS: + header += ( + " (Retrieval confidence is moderate — rely on these references " + "only where they clearly apply to the task.)" + ) + return header + "\n" + "\n".join(lines) + + # -- public -------------------------------------------------------- + def retrieve(self, query: str, k: Optional[int] = None) -> str: + """Return a CRAG-refined context block, or "" to fall back to baseline.""" + if not self.enabled or not query: + return "" + self.stats["calls"] += 1 + try: + hits = self._base.query_hits(query, k or self.k) + except Exception: + return "" + verdict = self._grade(self._scores(query, hits)) + + if verdict == INCORRECT: + rq = self._reformulate(query) + if rq == query: + self.stats["withheld"] += 1 + return "" # nothing to correct with -> withhold context + self.stats["rewrites"] += 1 + try: + hits2 = self._base.query_hits(rq, k or self.k) + except Exception: + hits2 = [] + verdict2 = self._grade(self._scores(rq, hits2)) if hits2 else INCORRECT + if verdict2 == INCORRECT: + self.stats["withheld"] += 1 + return "" # still weak -> withhold rather than mislead + hits, verdict, query = hits2, verdict2, rq + + refined = self._refine(query, hits) + if not refined: + return "" + return self._format(refined, verdict) diff --git a/agentigrid/agentigrid/rag/embed.py b/agentigrid/agentigrid/rag/embed.py new file mode 100755 index 00000000..b303ff81 --- /dev/null +++ b/agentigrid/agentigrid/rag/embed.py @@ -0,0 +1,27 @@ +"""Embed text via a local Ollama embedding model (default: nomic-embed-text).""" +from __future__ import annotations + +import requests + + +def embed( + text: str, + host: str = "http://localhost:11434", + model: str = "nomic-embed-text", + timeout: int = 60, +) -> list[float]: + """Return the embedding vector for `text` from an Ollama server. + + Args: + text: The text to embed. + host: Ollama base URL (e.g. http://localhost:11434). + model: Embedding model name (must be pulled in Ollama). + timeout: Request timeout in seconds. + """ + r = requests.post( + f"{host}/api/embeddings", + json={"model": model, "prompt": text}, + timeout=timeout, + ) + r.raise_for_status() + return r.json()["embedding"] diff --git a/agentigrid/agentigrid/rag/grader_reranker.py b/agentigrid/agentigrid/rag/grader_reranker.py new file mode 100644 index 00000000..7d27d98c --- /dev/null +++ b/agentigrid/agentigrid/rag/grader_reranker.py @@ -0,0 +1,128 @@ +"""Local cross-encoder relevance grader for Corrective RAG. + +Fills the grader seam of `CorrectiveRetriever`: + + grader(query, hits) -> [relevance score in 0..1 per hit] + +A cross-encoder reads the query and a retrieved chunk *together* and scores +their relevance, which is the standard stronger alternative to the embedding +cosine similarity used by the default grader. It runs entirely locally (no +network after the one-time model download), in line with the air-gapped design. + +Model: `AGENTIGRID_RERANKER_MODEL` (default ``BAAI/bge-reranker-base``, MIT +licensed). May be a Hugging Face id or a LOCAL DIRECTORY, so an air-gapped +machine can load a copied model. Optional `AGENTIGRID_RERANKER_DEVICE` +(e.g. ``cpu``, ``cuda``). + +Scores are mapped to (0, 1) with a sigmoid when the model returns raw logits. +They are relevance SCORES, not calibrated probabilities (calibration, e.g. Platt +scaling on labeled pairs, is future work). The corrective thresholds for this +grader should therefore be set from labeled examples, via +`AGENTIGRID_CRAG_TAU_LOWER` / `AGENTIGRID_CRAG_TAU_UPPER` (see rag/__init__.py). + +Design rules, as for every grader: + * generation-stage only — never touches engine/validation.py; + * failures are COUNTED and reported (describe()), so a run whose reranker fell + back to cosine is visible in the journal instead of being silently mislabeled. +""" +from __future__ import annotations + +import importlib.util +import math +import os +import time +from typing import Optional + +Hit = tuple[str, dict, float] +DEFAULT_MODEL = "BAAI/bge-reranker-base" + + +def reranker_available() -> bool: + """True if the cross-encoder library is installed (the model itself is + loaded, and may be downloaded, on first use).""" + return importlib.util.find_spec("sentence_transformers") is not None + + +def _sigmoid(x: float) -> float: + if x >= 0: + return 1.0 / (1.0 + math.exp(-x)) + z = math.exp(x) + return z / (1.0 + z) + + +def to_unit_interval(raw: list[float]) -> list[float]: + """Map scores into [0, 1]: keep them if already probabilities, else sigmoid.""" + vals = [float(x) for x in raw] + if all(0.0 <= v <= 1.0 for v in vals): + return vals + return [_sigmoid(v) for v in vals] + + +class RerankerGrader: + """Callable cross-encoder grader. Loads the model lazily on first use.""" + + def __init__(self, model_name: Optional[str] = None, device: Optional[str] = None, + batch_size: int = 16, _model=None): + self.model_name = model_name or os.environ.get("AGENTIGRID_RERANKER_MODEL") or DEFAULT_MODEL + self.device = device or os.environ.get("AGENTIGRID_RERANKER_DEVICE") or None + self.batch_size = batch_size + self._model = _model # injectable for tests + self._load_error: Optional[str] = None + self.calls = 0 + self.failures = 0 + self.seconds = 0.0 + + # -- model --------------------------------------------------------- + def _ensure_model(self): + if self._model is not None or self._load_error is not None: + return self._model + try: + from sentence_transformers import CrossEncoder + kwargs = {"device": self.device} if self.device else {} + self._model = CrossEncoder(self.model_name, **kwargs) + except Exception as exc: # missing library, no network, bad path ... + self._load_error = f"{type(exc).__name__}: {exc}"[:300] + self._model = None + return self._model + + @property + def available(self) -> bool: + return self._ensure_model() is not None + + # -- seam ---------------------------------------------------------- + def __call__(self, query: str, hits: list[Hit]) -> list[float]: + if not hits: + return [] + self.calls += 1 + model = self._ensure_model() + if model is None: + self.failures += 1 + raise RuntimeError(f"reranker unavailable: {self._load_error}") + pairs = [(query, doc) for (doc, _m, _s) in hits] + t0 = time.perf_counter() + try: + raw = model.predict(pairs, batch_size=self.batch_size, show_progress_bar=False) + except TypeError: # older/newer sentence-transformers signatures + raw = model.predict(pairs) + except Exception: + self.failures += 1 + raise + finally: + self.seconds += time.perf_counter() - t0 + scores = to_unit_interval(list(raw)) + if len(scores) != len(hits): + self.failures += 1 + raise RuntimeError("reranker returned a score list of the wrong length") + return scores + + def describe(self) -> dict: + ok = self._model is not None or (self._load_error is None and self.calls == 0) + name = "reranker" if ok and not self.failures else "reranker (fallback: cosine)" + return { + "grader": name, + "model": self.model_name, + "calls": self.calls, + "failures": self.failures, + "mean_latency_s": round(self.seconds / self.calls, 4) if self.calls else None, + "load_error": self._load_error, + } diff --git a/agentigrid/agentigrid/rag/ingest.py b/agentigrid/agentigrid/rag/ingest.py new file mode 100755 index 00000000..600c7e16 --- /dev/null +++ b/agentigrid/agentigrid/rag/ingest.py @@ -0,0 +1,88 @@ +"""Build the RAG index from a corpus folder. Run whenever the corpus changes. +Usage (from the agentigrid project root, venv active): + OLLAMA_HOST=http://localhost:11434 python -m agentigrid.rag.ingest rag/corpus + OLLAMA_HOST=... python -m agentigrid.rag.ingest rag/corpus_docs --store rag/store_docs +Reads .txt/.md files under the corpus dir, chunks them, embeds, and stores them +in a persistent Chroma index. The collection is REBUILT from scratch (no stale +chunks from removed files), and ingest_manifest.json in the store dir records the +corpus SHA-256 it was built from, so corpus_hash.corpus_status() can verify +that a store matches the frozen corpus it claims. +""" +from __future__ import annotations + +import os +import sys + +from .store import VectorStore + + +def chunk(text: str, size: int | None = None, overlap: int | None = None) -> list[str]: + """Paragraph-first chunking; only paragraphs longer than ``size`` are wrapped + (with ``overlap``). Defaults come from corpus_hash.CHUNKING, so every worked + example and schema exemplar (one paragraph each) stays a single chunk.""" + from .corpus_hash import CHUNKING + size = CHUNKING["size"] if size is None else size + overlap = CHUNKING["overlap"] if overlap is None else overlap + out: list[str] = [] + for para in text.split("\n\n"): + para = para.strip() + if not para: + continue + if len(para) <= size: + out.append(para) + else: + i = 0 + while i < len(para): + out.append(para[i : i + size]) + i += size - overlap + return out + + +def ingest( + corpus_dir: str = "rag/corpus", + store_path: str = "rag/store", + collection: str = "agentigrid_kb", + host: str = "http://localhost:11434", + model: str = "nomic-embed-text", +) -> None: + import datetime + import json + from pathlib import Path + + from .corpus_hash import CHUNKING, INGEST_MANIFEST_NAME, corpus_files, corpus_hash + + store = VectorStore(store_path, collection, host, model) + store.reset() + n, wrapped, longest = 0, 0, 0 + for p in corpus_files(Path(corpus_dir)): + path = str(p) + text = open(path, encoding="utf-8", errors="ignore").read() + for para in (q.strip() for q in text.split("\n\n")): + longest = max(longest, len(para)) + wrapped += len(para) > CHUNKING["size"] + for i, c in enumerate(chunk(text)): + store.add(f"{path}#{i}", c, {"source": os.path.basename(path)}) + n += 1 + digest, per_file = corpus_hash(Path(corpus_dir)) + Path(store_path, INGEST_MANIFEST_NAME).write_text(json.dumps({ + "corpus_dir": str(corpus_dir), "corpus_sha256": digest, "files": len(per_file), + "chunks": n, "collection": collection, "embed_model": model, + "chunking": {**CHUNKING, "wrapped_paragraphs": wrapped, "longest_paragraph_chars": longest}, + "ingested_at": datetime.datetime.now().isoformat(), + }, indent=2) + "\n") + print(f"Ingested {n} chunks from {len(per_file)} file(s); collection '{collection}' holds " + f"{store.count()}; corpus sha256 {digest}.") + print(f"Chunking {CHUNKING['scheme']}: longest paragraph {longest} chars; " + f"{wrapped} paragraph(s) longer than {CHUNKING['size']} were split.") + + +if __name__ == "__main__": + import argparse + + ap = argparse.ArgumentParser(description="Build the RAG index from a corpus folder.") + ap.add_argument("corpus", nargs="?", default="rag/corpus") + ap.add_argument("--store", default="rag/store") + ap.add_argument("--collection", default="agentigrid_kb") + a = ap.parse_args() + ingest(corpus_dir=a.corpus, store_path=a.store, collection=a.collection, + host=os.environ.get("OLLAMA_HOST", "http://localhost:11434")) diff --git a/agentigrid/agentigrid/rag/retriever.py b/agentigrid/agentigrid/rag/retriever.py new file mode 100755 index 00000000..29a1c82d --- /dev/null +++ b/agentigrid/agentigrid/rag/retriever.py @@ -0,0 +1,132 @@ +"""Retrieval step for the GENERATION stage only. + +Design rule: retrieval grounds the LLM's spec/proposal generation. It must +never be consulted by the deterministic semantic validator — keep it out of +engine/validation.py so verification stays deterministic and reproducible. + +`retrieve()` is defensive: any failure (Ollama down, empty index, missing +chromadb) returns "" so a run behaves exactly like the no-RAG baseline. + +Relevance is filtered by a cosine-similarity threshold (`min_score`) so weak +matches are dropped instead of always returning `k` chunks. +""" +from __future__ import annotations + +import logging +from typing import Optional + +logger = logging.getLogger("agentigrid.rag") + + +class Retriever: + def __init__( + self, + enabled: bool = True, + path: str = "rag/store", + collection: str = "agentigrid_kb", + host: str = "http://localhost:11434", + model: str = "nomic-embed-text", + k: int = 3, + min_score: float = 0.35, + ): + self.enabled = enabled + # Per-session retrieval telemetry (journaled via describe()): a failed + # query (e.g. embedding server down) silently degrades to the no-RAG + # baseline, so errors must be counted or a retrieval run is mislabelled. + self.stats = {"calls": 0, "errors": 0, "empty": 0} + self.k = k + self.min_score = min_score + self.store_path = path + self.collection = collection + self._store = None + if enabled: + try: + from .store import VectorStore + + self._store = VectorStore(path, collection, host, model) + except Exception: + self.enabled = False # degrade gracefully to baseline + + def retrieve(self, query: str, k: Optional[int] = None) -> str: + """Return a formatted context block for `query`, or "" if disabled/empty. + + Only chunks with cosine similarity >= min_score are kept. + """ + if not self.enabled or self._store is None or not query: + return "" + self.stats["calls"] += 1 + try: + hits = self._store.query(query, k or self.k) + except Exception as exc: + self.stats["errors"] += 1 + logger.warning("Retrieval failed (%s: %s); this iteration runs without references", + type(exc).__name__, exc) + return "" # retrieval must never break a run + hits = [(doc, meta, score) for (doc, meta, score) in hits if score >= self.min_score] + if not hits: + self.stats["empty"] += 1 + return "" + lines = [ + f"[ref {i + 1} | score {score:.2f}] {doc}" + for i, (doc, _meta, score) in enumerate(hits) + ] + return ( + "Use the following retrieved reference material only when relevant; " + "do not invent facts beyond it.\n" + "\n".join(lines) + ) + + def describe(self) -> dict: + """Effective configuration for this run (shown in the UI / logged).""" + return { + "mode": "basic", + "enabled": self.enabled, + "k": self.k, + "min_score": self.min_score, + "store_path": self.store_path, + "collection": self.collection, + "stats": dict(self.stats), + } + + def query_hits(self, query: str, k: Optional[int] = None) -> list[tuple[str, dict, float]]: + """Raw scored hits (document, metadata, similarity), best first, with NO + ``min_score`` filtering. Advanced modes (e.g. Corrective RAG) grade these + with their own thresholds. Returns [] on disabled/empty/any failure. + """ + if not self.enabled or self._store is None or not query: + return [] + try: + return self._store.query(query, k or self.k) + except Exception as exc: + self.stats["errors"] += 1 + logger.warning("Retrieval failed (%s: %s)", type(exc).__name__, exc) + return [] + + def embed_text(self, text: str) -> list[float]: + """Embed `text` with the same Ollama host/model as the store, for + strip-level re-scoring in advanced modes. Raises on failure (callers + that must degrade gracefully wrap this in try/except). + """ + from .embed import embed as _embed + + return _embed(text, self._store._host, self._store._model) + + @classmethod + def from_config(cls, cfg) -> "Retriever": + """Build a Retriever from the app config's optional `rag:` block. + + Falls back to llm.ollama_host so the embedder reaches the same Ollama as + the chat backend. ADAPT the attribute access to AgentiGrid's config object. + """ + rag = getattr(cfg, "rag", None) + get = (rag.get if isinstance(rag, dict) else (lambda k, d=None: getattr(rag, k, d))) if rag else (lambda k, d=None: d) + llm = getattr(cfg, "llm", None) + llm_host = getattr(llm, "ollama_host", None) if llm else None + return cls( + enabled=bool(get("enabled", False)), + path=get("store_path", "rag/store"), + collection=get("collection", "agentigrid_kb"), + host=get("ollama_host", None) or llm_host or "http://localhost:11434", + model=get("embed_model", "nomic-embed-text"), + k=int(get("top_k", 3)), + min_score=float(get("min_score", 0.35)), + ) diff --git a/agentigrid/agentigrid/rag/store.py b/agentigrid/agentigrid/rag/store.py new file mode 100755 index 00000000..e47f9d21 --- /dev/null +++ b/agentigrid/agentigrid/rag/store.py @@ -0,0 +1,76 @@ +"""Persistent vector store backed by Chroma. + +This is the only file that talks to the vector database. Swap Chroma for +FAISS/pgvector here later without touching the rest of the RAG package. + +Uses cosine distance so query() can return an interpretable similarity in +[0, 1] (1 = identical meaning), which the retriever thresholds on. +NOTE: changing the distance space requires a fresh index — delete rag/store +and re-run ingest after upgrading to this version. +""" +from __future__ import annotations + +from .embed import embed + + +class VectorStore: + def __init__( + self, + path: str = "rag/store", + collection: str = "agentigrid_kb", + host: str = "http://localhost:11434", + model: str = "nomic-embed-text", + ): + import chromadb # imported lazily so the package loads even without chromadb + + self._client = chromadb.PersistentClient(path=path) + # cosine space -> distance in [0, 2]; similarity = 1 - distance + self._col = self._client.get_or_create_collection( + collection, metadata={"hnsw:space": "cosine"} + ) + self._host, self._model = host, model + + def add(self, doc_id: str, text: str, metadata: dict | None = None) -> None: + self._col.add( + ids=[doc_id], + embeddings=[embed(text, self._host, self._model)], + documents=[text], + metadatas=[metadata or {}], + ) + + def query(self, text: str, k: int = 5) -> list[tuple[str, dict, float]]: + """Return [(document, metadata, similarity), ...], best first. + + similarity = 1 - cosine_distance, so ~1.0 is a very close match and + values near 0 (or below) are weak/unrelated. + """ + res = self._col.query( + query_embeddings=[embed(text, self._host, self._model)], + n_results=k, + include=["documents", "metadatas", "distances"], + ) + docs = res.get("documents", [[]])[0] + metas = res.get("metadatas", [[]])[0] + dists = res.get("distances", [[]])[0] + out = [] + for doc, meta, dist in zip(docs, metas, dists): + out.append((doc, meta, 1.0 - float(dist))) + return out + + def count(self) -> int: + return self._col.count() + + def all_documents(self) -> list[tuple[str, str, dict]]: + """Every stored chunk as (id, document, metadata) -- for offline tools + such as relevance labelling; not used on the retrieval path.""" + res = self._col.get(include=["documents", "metadatas"]) + return list(zip(res.get("ids", []), res.get("documents", []), res.get("metadatas", []))) + + def reset(self) -> None: + """Drop and recreate the collection (a fresh index, no stale chunks).""" + name = self._col.name + try: + self._client.delete_collection(name) + except Exception: + pass + self._col = self._client.get_or_create_collection(name, metadata={"hnsw:space": "cosine"}) diff --git a/agentigrid/claude_code_changes.md b/agentigrid/claude_code_changes.md index 3c0fed69..1384f98a 100644 --- a/agentigrid/claude_code_changes.md +++ b/agentigrid/claude_code_changes.md @@ -4,6 +4,56 @@ This document records significant changes made by Claude Code, grouped by the pr --- +## Optional retrieval grounding (RAG) for LLM proposals (2026-10-06) + +**What.** AgentiGrid can add a short block of reference material, retrieved from a curated corpus, to the LLM's per-iteration prompt. It is **off by default**: with RAG off, behaviour and dependencies are unchanged. Retrieval touches only the generation step. The deterministic validator never sees it, and any retrieval failure leaves the prompt unchanged, so the run behaves like the baseline. + +### Package `agentigrid/rag/` + +| File | Role | +|---|---| +| `__init__.py` | `build_retriever()` chooses the mode from `AGENTIGRID_RAG_MODE` (`off` / `basic` / `corrective`; legacy `AGENTIGRID_RAG=1` means `basic`), the grader and the thresholds | +| `retriever.py` | Basic mode: top `k=3` chunks with cosine similarity of at least `0.35`. Counts calls, errors and empty results | +| `corrective.py` | Corrective mode: grade hits (CORRECT / AMBIGUOUS / INCORRECT), refine to relevant sentences, reformulate once, withhold context when still INCORRECT | +| `grader_reranker.py` | Optional local cross-encoder grader (`sentence-transformers`). Failures fall back to cosine and are counted | +| `embed.py`, `store.py` | Ollama embeddings (`nomic-embed-text`) and a persistent Chroma index; `chromadb` is imported lazily | +| `ingest.py`, `corpus_hash.py` | Build the index; record the corpus SHA-256 and chunking scheme; `corpus_status()` checks a store against the frozen corpus | + +### Integration + +- **`exago_engine/agent_loop.py`.** Builds the retriever at start-up. Before each per-iteration LLM call it retrieves with the goal text and prepends "Section B: Reference Material (retrieved)". It emits a `rag_retrieved::` phase when RAG is on, and records `rag_enabled` / `rag_config` (mode, thresholds, grader, call/error counts) at the end of the run. Post-search analysis is not grounded. +- **`exago_engine/journal.py`.** Exports `rag_enabled` and `rag_config`. +- **`launcher/app.py`, `launcher/session_manager.py`.** "Reference knowledge (RAG)" selector under **Advanced** (default **Off**). The relevance grader choice appears for corrective mode, and the reranker is offered only when `sentence-transformers` is installed. A grounding caption shows on the live monitor and on the results page. + +### Corpus and tools + +- **`rag/corpus/` (frozen v1, 12 files, manifest SHA-256 `da4b77a5…`).** ExaGO `--help` text per application, case metadata, methodology notes, certified schema exemplars, and solver-certified worked examples from successful runs. +- **`rag/tools/rag_harvest.py`.** Generates tool help and case metadata from the binaries and `.m` files. +- **`rag/tools/rag_schema_exemplars.py`.** Builds response examples, certified by AgentiGrid's own parser, validator and modifier on `case118`. +- **`rag/tools/corpus_guard.py`.** Hold-out leakage and personal-path audit, plus `--freeze`. + +### Dependencies + +`requirements-rag.txt` (optional): `chromadb`, `requests`, and `sentence-transformers` (commented out) for the reranker. The core `requirements.txt` is unchanged. + +### Tests + +`tests/rag/` has 23 tests, using fake embeddings and fake models, so no Ollama, Chroma or model download is needed: + +- off by default; +- references reach the prompt and the journal; +- a failed retrieval leaves the prompt unchanged; +- retrieval errors are counted; +- the corrective grader contract; +- the chunking scheme; +- the shipped corpus matches its manifest and contains no personal paths; +- the schema exemplars certify on `case118`; +- the guard flags. + +With chromadb installed and a fake embedder: ingest of `rag/corpus` gives 92 chunks and the manifest hash; basic returns 3 references, corrective returns refined strips, and off returns nothing. + +--- + ## Agent loop and LLM backends: robust iterations, honest UI, usage accounting (2026-10-03) **Motivation.** Long runs with local models (Ollama) showed iterations that went wrong in ways the UI hid or misreported: a malformed action crashed the run; a rejected iteration re-emitted the previous card, so the timeline showed stale or out-of-order cards (e.g. "Iteration 6" after iteration 20); analysis and completion entries were labelled FAILED; a modify that applied no command showed "No description"; and the summary counted the base case as an iteration ("Iterations 3" for max 2). Separately, Anthropic runs re-sent the same system prompt at full price on every iteration, and some newer models reject the `temperature` parameter. diff --git a/agentigrid/docs/RAG_USAGE.md b/agentigrid/docs/RAG_USAGE.md new file mode 100755 index 00000000..d32310ae --- /dev/null +++ b/agentigrid/docs/RAG_USAGE.md @@ -0,0 +1,104 @@ +# RAG in AgentiGrid: optional retrieval grounding + +## What it is + +Retrieval-augmented generation (RAG) adds a short block of reference material to the LLM's prompt. The material comes from a curated corpus in `rag/corpus/`: ExaGO tool help, case metadata, methodology notes, and certified examples of correct AgentiGrid responses. The aim is fewer proposals that are executable but wrong for the goal. + +- **Off by default.** With RAG off, AgentiGrid behaves exactly as before and needs none of the extra dependencies. +- **Generation stage only.** Retrieval changes what the LLM sees. It never reaches the deterministic validator (`engine/validation.py`), so verification stays reproducible. +- **Defensive.** Any retrieval failure (embedding server down, empty index, missing `chromadb`) returns nothing, and the run behaves like the no-RAG baseline. Failures are counted in the journal (`rag_config.stats`), so a degraded run is visible rather than mislabelled. + +## Modes + +The environment variable `AGENTIGRID_RAG_MODE` selects the mode. The UI sets it from the "Reference knowledge (RAG)" selector under **Advanced**. + +| Mode | What reaches the prompt | +|---|---| +| `off` (default) | Nothing | +| `basic` | Up to `k=3` chunks with cosine similarity of at least `min_score=0.35` | +| `corrective` | Retrieved chunks are graded CORRECT, AMBIGUOUS or INCORRECT, then refined to the relevant sentences. On INCORRECT the query is reformulated once; if still INCORRECT, the context is withheld. | + +The older switch `AGENTIGRID_RAG=1` / `0` still works and means `basic` / `off`. + +**Corrective mode options:** + +- `AGENTIGRID_CRAG_GRADER`: `cosine` (default; deterministic) or `reranker`. The reranker is a local cross-encoder. It needs `sentence-transformers`; the model is set with `AGENTIGRID_RERANKER_MODEL`, default `BAAI/bge-reranker-base`. +- `AGENTIGRID_CRAG_TAU_LOWER` / `AGENTIGRID_CRAG_TAU_UPPER`: grading thresholds, default 0.30 / 0.50. These are starting points, not calibrated values. + +**Store selection:** `AGENTIGRID_RAG_STORE` and `AGENTIGRID_RAG_COLLECTION` point at a different index, for example one built from another corpus. + +## Code layout + +- `agentigrid/rag/`, the importable package `agentigrid.rag`: + - `embed.py`: Ollama embeddings (`nomic-embed-text`); + - `store.py`: persistent Chroma index, cosine distance; + - `retriever.py`: basic mode; + - `corrective.py`: corrective mode; + - `grader_reranker.py`; + - `ingest.py`: builds the index; + - `corpus_hash.py`: corpus identity and chunking scheme. +- `rag/corpus/`: the source `.txt` / `.md` files and `corpus_manifest.json` (the frozen corpus hash). +- `rag/store/`: the built index. It's gitignored; build it locally. +- `rag/tools/`: corpus builders and the audit tool (see `rag/tools/README.md`). + +## Setup + +All commands run from the AgentiGrid root (`agentigrid/`). + +1. Install the optional dependencies: + + ```bash + pip install -r requirements-rag.txt + ``` + +2. Run Ollama with the embedding model, and point AgentiGrid at it if it isn't on localhost: + + ```bash + ollama pull nomic-embed-text + export OLLAMA_HOST="http://:11434" + ``` + +3. Build the index. Re-run this after any corpus change. + + ```bash + rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus + ``` + + It prints the number of chunks and records the corpus hash and chunking scheme in `rag/store/ingest_manifest.json`. Chunking keeps each paragraph whole up to 3000 characters, so worked examples are never cut in half. + +## Running + +**CLI:** + +```bash +GOAL="Reduce total generation cost by 10%" +agentigrid ./data/exago/examples/case39.m "$GOAL" --backend ollama --model qwen2.5:7b --max-iter 4 # off +AGENTIGRID_RAG_MODE=basic agentigrid ./data/exago/examples/case39.m "$GOAL" --backend ollama --model qwen2.5:7b --max-iter 4 # basic +``` + +**UI:** `./launcher/run.sh`, then go to **Advanced**, then **Reference knowledge (RAG)**. During a search, the live monitor shows a grounding caption with the mode, the number of references and the top score. The results page states whether the run used RAG. + +**Check retrieval on its own:** + +```bash +python -c "import os; from agentigrid.rag import Retriever; print(Retriever(enabled=True, host=os.environ.get('OLLAMA_HOST', 'http://localhost:11434')).retrieve('Assess security under single-element outages'))" +``` + +Lines like `[ref N | score 0.xx]` mean it works. Empty output means see Troubleshooting. + +## What a run records + +`journal.json` gets two fields: + +- `rag_enabled`; +- `rag_config`: the mode, `k`, `min_score`, the thresholds and grader for corrective mode, and the counts of retrieval calls, errors and empty results. + +## Troubleshooting + +- **Empty retrieval with RAG on.** There are three likely causes: + - `OLLAMA_HOST` is unset or wrong; + - the index is empty or was built with another chunking scheme; rebuild it with `rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus`; + - every hit is below `min_score`. +- **`enabled: false` in `rag_config` although a mode was set.** The retriever disabled itself at start-up: either `chromadb` is missing or the store path is wrong. +- **`model ... not found (404)` from embeddings.** Run `ollama pull nomic-embed-text`. +- **Reranker not offered in the UI.** `sentence-transformers` isn't installed. diff --git a/agentigrid/launcher/app.py b/agentigrid/launcher/app.py index 2711e738..36e645d8 100644 --- a/agentigrid/launcher/app.py +++ b/agentigrid/launcher/app.py @@ -172,6 +172,10 @@ def start_search(base_case_path, goal, backend, model, temperature, st.session_state.steering_history = [] st.session_state.search_paused = False st.session_state.explore_status = None + st.session_state.rag_last_refs = None + st.session_state.rag_run_config = None + st.session_state.rag_run_enabled = os.environ.get("AGENTIGRID_RAG") == "1" + st.session_state.rag_run_mode = os.environ.get("AGENTIGRID_RAG_MODE") or ("basic" if st.session_state.rag_run_enabled else "off") try: manager.start_search(config_overrides=overrides, goal=goal) @@ -224,6 +228,70 @@ def render_sidebar() -> dict: disabled=disabled, ) + # ── Advanced ───────────────────────────────────────────────────── + with st.expander("🔧 Advanced"): + rag_mode = st.radio( + "Reference knowledge (RAG)", + options=["off", "basic", "corrective"], + index=0, # default: off (RAG needs optional dependencies and an embedding server) + format_func=lambda m: { + "off": "Off — no grounding", + "basic": "Basic", + "corrective": "Corrective (experimental)", + }[m], + horizontal=True, + disabled=disabled, + help=( + "Ground the AI's proposals in a curated knowledge base via retrieval. " + "Basic returns the top matches above a similarity threshold. " + "Corrective grades the retrieval and withholds weak context " + "(needs threshold calibration — see the note when selected)." + ), + ) + if rag_mode == "corrective": + st.info( + "**Corrective RAG — calibrate before trusting results.**\n\n" + "1. Build + ingest the corpus first (harvest tools → `ingest`).\n" + "2. Run a few searches and watch the grounding indicator's scores " + "to see your similarity distribution.\n" + "3. Set thresholds in `rag/corrective.py`: `tau_upper` (score treated " + "as \"correct\"), `tau_lower` (below → \"incorrect\", context withheld), " + "`strip_min_score` (refinement cutoff). Defaults 0.50 / 0.30 / 0.30 are " + "**starting points, not calibrated**.\n\n" + "Corrective withholds context on low confidence, so on a thin or " + "uncalibrated corpus it may behave like the baseline — that is expected." + ) + # Relevance grader (corrective only). The reranker is offered only + # when sentence-transformers is installed, so a run can never claim + # a grader that didn't run. + import importlib.util + _graders = ["cosine"] + if importlib.util.find_spec("sentence_transformers") is not None: + _graders.append("reranker") + crag_grader = st.radio( + "Relevance grader", + options=_graders, + index=0, + format_func=lambda g: { + "cosine": "Cosine (deterministic)", + "reranker": "Reranker (local cross-encoder)", + }[g], + horizontal=True, + disabled=disabled, + help=( + "How Corrective RAG grades retrieved chunks. Cosine is the " + "reproducible default. The reranker scores each (goal, chunk) " + "pair with a local cross-encoder (needs sentence-transformers)." + ), + ) + else: + crag_grader = "cosine" + # AGENTIGRID_RAG_MODE is the mode axis; keep the legacy AGENTIGRID_RAG flag + # in sync so the existing grounding indicator (which checks it) still works. + os.environ["AGENTIGRID_RAG_MODE"] = rag_mode + os.environ["AGENTIGRID_RAG"] = "0" if rag_mode == "off" else "1" + os.environ["AGENTIGRID_CRAG_GRADER"] = crag_grader + # ── Search Parameters ──────────────────────────────────────────── st.header("⚙️ Search Parameters") application = st.selectbox( @@ -756,6 +824,7 @@ def render_live_monitor(): st.session_state.base_opflow = manager.get_base_opflow() st.session_state.best_opflow = manager.get_best_opflow() st.session_state.goal_classification = manager.get_goal_classification() + st.session_state.rag_run_config = manager.get_rag_config() # Append to session history session = manager.get_session() if session: @@ -772,20 +841,27 @@ def render_live_monitor(): st.session_state.iteration_log.append(update) st.session_state.current_iteration = update["iteration"] elif update["type"] == "phase": - phase_labels = { - "llm_request": "Sending prompt to LLM...", - "applying_commands": "Applying modifications...", - "running_simulation": "Running simulation...", - "parsing_results": "Parsing results...", - "computing_pareto_front": "Computing Pareto front...", - } raw_phase = update["phase"] - if raw_phase.startswith("running_simulation ("): - st.session_state.current_phase = f"Running {raw_phase.split('(')[1].rstrip(')')}..." + + if raw_phase.startswith("rag_retrieved:"): + _parts = raw_phase.split(":") + try: + st.session_state.rag_last_refs = int(_parts[1]) + st.session_state.rag_top_score = float(_parts[2]) if len(_parts) > 2 else None + except (ValueError, IndexError): + pass else: - st.session_state.current_phase = phase_labels.get( - raw_phase, raw_phase - ) + phase_labels = { + "llm_request": "Sending prompt to LLM...", + "applying_commands": "Applying modifications...", + "running_simulation": "Running simulation...", + "parsing_results": "Parsing results...", + "computing_pareto_front": "Computing Pareto front...", + } + if raw_phase.startswith("running_simulation ("): + st.session_state.current_phase = f"Running {raw_phase.split('(')[1].rstrip(')')}..." + else: + st.session_state.current_phase = phase_labels.get(raw_phase, raw_phase) elif update["type"] == "pause_state": st.session_state.search_paused = update["paused"] elif update["type"] == "explore_status": @@ -802,6 +878,7 @@ def render_live_monitor(): st.session_state.base_opflow = manager.get_base_opflow() st.session_state.best_opflow = manager.get_best_opflow() st.session_state.goal_classification = manager.get_goal_classification() + st.session_state.rag_run_config = manager.get_rag_config() if not st.session_state.search_error: st.session_state.search_error = "Search thread terminated unexpectedly." st.rerun() @@ -810,6 +887,29 @@ def render_live_monitor(): # 2. Header st.header("🔄 Search in Progress...") + _rag_mode = os.environ.get("AGENTIGRID_RAG_MODE") or ("basic" if os.environ.get("AGENTIGRID_RAG") == "1" else "off") + _rag_on = _rag_mode != "off" + _refs = st.session_state.get("rag_last_refs") + _top = st.session_state.get("rag_top_score") + if _rag_on: + # Pull the effective retriever config (thresholds etc.) from the running + # controller so Live shows the same values as the Search-Complete page. + if not st.session_state.get("rag_run_config") and manager is not None: + st.session_state.rag_run_config = manager.get_rag_config() + _cfg = st.session_state.get("rag_run_config") or {} + if _refs is not None: + _s = f" · top {_top:.2f}" if _top else "" + # For corrective, 0 refs means context was withheld (low confidence). + st.caption(f"🔎 Grounding: {_rag_mode} · {_refs} ref(s){_s}") + else: + st.caption(f"🔎 Grounding: {_rag_mode} · enabled") + if _cfg.get("mode") == "corrective": + st.caption( + f"k={_cfg.get('k')}, min_score={_cfg.get('min_score')}, " + f"τ_lower={_cfg.get('tau_lower')}, τ_upper={_cfg.get('tau_upper')}, " + f"strip_min={_cfg.get('strip_min_score')}, grader={_cfg.get('grader')}" + ) + # 3. Two-column layout left_col, right_col = st.columns([2, 1]) @@ -1112,6 +1212,21 @@ def render_results(): return st.header("✅ Search Complete") + _rag_used = getattr(session, "rag_enabled", None) + if _rag_used is None: + _rag_used = st.session_state.get("rag_run_enabled") + if _rag_used: + _cfg = st.session_state.get("rag_run_config") or {} + _mode = _cfg.get("mode", st.session_state.get("rag_run_mode", "basic")) + if _mode == "corrective": + _details = ( + f" — k={_cfg.get('k')}, min_score={_cfg.get('min_score')}, " + f"τ_lower={_cfg.get('tau_lower')}, τ_upper={_cfg.get('tau_upper')}, " + f"strip_min={_cfg.get('strip_min_score')}, grader={_cfg.get('grader')}" + ) + else: + _details = "" # basic/off: the fixed defaults aren't worth showing + st.caption(f"🔎 This run used RAG grounding (mode: {_mode}){_details}.") tab1, tab2, tab3 = st.tabs([ "📊 Overview", "🔍 Detailed Results", "📝 Analysis & Report", diff --git a/agentigrid/launcher/session_manager.py b/agentigrid/launcher/session_manager.py index bb691eaf..9f1b1625 100644 --- a/agentigrid/launcher/session_manager.py +++ b/agentigrid/launcher/session_manager.py @@ -166,6 +166,17 @@ def get_goal_classification(self) -> dict | None: """Get the LLM-determined goal classification, or None if not yet computed.""" return self._goal_classification + def get_rag_config(self) -> dict | None: + """Effective RAG configuration for the run (mode + thresholds), or None.""" + if self._controller is not None and hasattr(self._controller, "_retriever"): + r = self._controller._retriever + if hasattr(r, "describe"): + try: + return r.describe() + except Exception: + return None + return None + def get_opflow_by_iteration(self, iteration: int) -> OPFLOWResult | None: """Get the OPFLOW result for a specific iteration.""" return self._opflow_by_iteration.get(iteration) diff --git a/agentigrid/rag/corpus/agentigrid_schema_exemplars.txt b/agentigrid/rag/corpus/agentigrid_schema_exemplars.txt new file mode 100644 index 00000000..dfedee17 --- /dev/null +++ b/agentigrid/rag/corpus/agentigrid_schema_exemplars.txt @@ -0,0 +1,72 @@ +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow] +Request: Set voltage limits on every bus to 0.97-1.03 pu. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "All-bus voltage band 0.97-1.03 pu", "reasoning": "A system-wide band is one set_all_bus_vlimits command, nested in modify.", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Note: set_all_bus_vlimits is a COMMAND, not an action. Wrong: {"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03} as the whole response. Right: nest it in the "commands" list of a "modify" action. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow] +Request: Tighten the voltage limits of bus 12 only to 0.98-1.02 pu. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Bus 12 voltage band 0.98-1.02 pu", "reasoning": "Single-bus limits use set_bus_vlimits.", "commands": [{"action": "set_bus_vlimits", "bus": 12, "Vmin": 0.98, "Vmax": 1.02}]} +Note: In OPFLOW/SCOPFLOW, bus voltages are decision variables: constrain them with set_bus_vlimits / set_all_bus_vlimits, not with set_gen_voltage. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Increase all loads in the network by 8%. +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "Uniform load scaling x1.08", "reasoning": "Uniform scaling of every load is scale_all_loads with factor 1.08.", "commands": [{"action": "scale_all_loads", "factor": 1.08}]} +Note: factor is a multiplier: 1.08 = +8%, 0.9 = -10%. mode "fresh" applies the change to the base case; "accumulative" stacks it on the current state. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Set the load at bus 15 to 95 MW and 30 MVAr. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Bus 15 load 95 MW / 30 MVAr", "reasoning": "Absolute load values use set_load.", "commands": [{"action": "set_load", "bus": 15, "Pd": 95.0, "Qd": 30.0}]} + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Add a new 40 MW load at bus 20 on top of its existing load. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Add 40 MW at bus 20", "reasoning": "add_load_at_bus adds to the existing load; set_load would replace it.", "commands": [{"action": "add_load_at_bus", "bus": 20, "Pd": 40.0, "Qd": 0.0}]} + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Take the generator at bus 10 out of service. +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "Generator at bus 10 offline", "reasoning": "Generator outage is set_gen_status with status 0.", "commands": [{"action": "set_gen_status", "bus": 10, "status": 0}]} + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: pflow] +Request: Set the output of the generator at bus 12 to 150 MW. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Bus 12 generator Pg = 150 MW", "reasoning": "In PFLOW the dispatch is fixed by set_gen_dispatch; Pg must lie in [Pmin, Pmax].", "commands": [{"action": "set_gen_dispatch", "bus": 12, "Pg": 150.0}]} +Note: In OPFLOW/SCOPFLOW the optimizer re-dispatches generators; to force an output there, bound it instead (this command sets Pg as the starting value). + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: pflow] +Request: Raise the voltage setpoint of the generator at bus 10 to 1.04 pu. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Bus 10 Vg = 1.04 pu", "reasoning": "In PFLOW the generator voltage setpoint is enforced, so it is the primary voltage control.", "commands": [{"action": "set_gen_voltage", "bus": 10, "Vg": 1.04}]} +Note: Only meaningful in PFLOW. In OPFLOW/SCOPFLOW the solver overrides Vg. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: pflow] +Request: Add 25 MVAr of capacitive shunt support at bus 44. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Bus 44 shunt Bs = 25", "reasoning": "Positive Bs is capacitive and raises the local voltage.", "commands": [{"action": "set_shunt_susceptance", "bus": 44, "Bs": 25.0}]} + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: pflow] +Request: Change the tap ratio of the transformer between buses 8 and 5 to 1.0. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Transformer 8-5 tap = 1.0", "reasoning": "Tap changes use set_tap_ratio on a transformer branch.", "commands": [{"action": "set_tap_ratio", "fbus": 8, "tbus": 5, "ratio": 1.0}]} +Note: Only transformer branches (ratio != 0 in the case) accept a tap change. + +[source: schema exemplar | generated from AgentiGrid's command set | schema-validated only (not applicable to reference case: Branch 30-17 is not a phase shifter (angle is 0)) | apps: pflow] +Request: Set the phase shifter between buses 30 and 17 to 4 degrees. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Phase shift 30-17 = 4 deg", "reasoning": "Phase-shifter angle changes use set_phase_shift_angle.", "commands": [{"action": "set_phase_shift_angle", "fbus": 30, "tbus": 17, "angle": 4.0}]} +Note: Only branches that are phase shifters (angle != 0 in the case) accept this command. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Take the line between buses 23 and 24 out of service. +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "Line 23-24 out of service", "reasoning": "A branch outage is set_branch_status with status 0.", "commands": [{"action": "set_branch_status", "fbus": 23, "tbus": 24, "status": 0}]} +Note: In SCOPFLOW this permanently changes the topology; contingencies come from the contingency file, not from set_branch_status. + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow, pflow] +Request: Raise the thermal rating of the line between buses 30 and 38 to 250 MVA. +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "Line 30-38 rateA = 250 MVA", "reasoning": "Branch ratings use set_branch_rate (rateA in MVA).", "commands": [{"action": "set_branch_rate", "fbus": 30, "tbus": 38, "rateA": 250.0}]} + +[source: schema exemplar | generated from AgentiGrid's command set | validated: parsed and applied on held-out case118 | apps: opflow, scopflow] +Request: Scale all loads by 1.05 and set the voltage band on every bus to 0.96-1.04 pu in one step. +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "Loads x1.05 with 0.96-1.04 pu band", "reasoning": "Several commands go in one commands list, applied in order.", "commands": [{"action": "scale_all_loads", "factor": 1.05}, {"action": "set_all_bus_vlimits", "Vmin": 0.96, "Vmax": 1.04}]} + +[source: schema exemplar | generated from AgentiGrid's command set | structure-validated | apps: opflow, scopflow, pflow] +Request: List the 3 buses nearest to bus 49 before deciding. +Correct response (JSON): {"action": "analyze", "query_type": "nearest_neighbors", "bus": 49, "k": 3, "reasoning": "Topology questions use a structured query_type, not free text."} + +[source: schema exemplar | generated from AgentiGrid's command set | structure-validated | apps: opflow, scopflow, pflow] +Request: The target is met; stop and report. +Correct response (JSON): {"action": "complete", "reasoning": "The last feasible solve satisfies the stated target.", "findings": {"summary": "One-sentence answer to the goal, with the key number.", "details": "Iteration, objective value and constraint status supporting it."}} +Note: complete carries findings; it does not carry commands. diff --git a/agentigrid/rag/corpus/corpus_manifest.json b/agentigrid/rag/corpus/corpus_manifest.json new file mode 100644 index 00000000..81b8b5cc --- /dev/null +++ b/agentigrid/rag/corpus/corpus_manifest.json @@ -0,0 +1,21 @@ +{ + "corpus_sha256": "da4b77a512df7b47230d8d2ded9610077bc8b35612d61c131c4e2d14d14ced20", + "files": { + "agentigrid_schema_exemplars.txt": "57cfc8cea7a83dde72fd149321376310491db7fc2d8efde786cfe4332a0476a3", + "exago_cases.txt": "d6acee84a70e58f85a9d58f911e8581b22d2a5659467c89e93c85322e91fdfeb", + "exago_dcopflow_help.txt": "2aaeee330ebe9b8a693a3bce814f8bcaa0f8f45fb0c7e779188c672cce2ce481", + "exago_exemplars_from_runs.txt": "9ae1edcf11671e0403b4cf04a919a8fb29bd2e8b0ac651a75328b61383471899", + "exago_opflow_help.txt": "3f87ccf31fd8d11a34037ba42e6efd86e95a26576db2fcded5a04598dc817758", + "exago_pflow_help.txt": "a448c025037758950f699bfdf3473f87304230fb2904cc2135735a2f9886b8fc", + "exago_scopflow_help.txt": "17f4ad66bc59bc5c97e434134e448d4010749ab4ca7ed5c67c4fb2a99031fedd", + "exago_sopflow_help.txt": "66de2205df565ffd560e4aee6e7ba62c758045e8d3750cf241a3d7e170278f7f", + "exago_tcopflow_help.txt": "73961625b93c816c558d9656a07c6e0ff4b71ed570ebef4f711d879066cbcdc5", + "example_scopf_spec.txt": "ab7a70834d25f2d4a902759de2558f2e81eb2e76c1d824cdaab3f8bc5d6caec1", + "n1_methodology.txt": "61645ed9d14db276b90d23b6641e1341950de4a84ecff7a8418944c68c43a774", + "opflow_help.txt": "b4aa21996a0f2c09fd770112d90d85ec477082f94c304ce3dc12a045cf4115e1" + }, + "held_out_specs": [ + "grader_ablation_spec.json" + ], + "frozen_at": "2026-09-28T12:30:52.587379" +} diff --git a/agentigrid/rag/corpus/exago_cases.txt b/agentigrid/rag/corpus/exago_cases.txt new file mode 100644 index 00000000..1781a7d1 --- /dev/null +++ b/agentigrid/rag/corpus/exago_cases.txt @@ -0,0 +1,8 @@ +[source: case file: data/case39.m | harvested: 2026-09-07 | auto-generated, review before trusting] +case39.m: 39 buses, 10 generators, 46 branches. MATPOWER-format network case usable with ExaGO OPF apps (pass via the app's network/case option). + +[source: case file: data/case9modalt.m | harvested: 2026-09-07 | auto-generated, review before trusting] +case9modalt.m: 9 buses, 3 generators, 9 branches. MATPOWER-format network case usable with ExaGO OPF apps (pass via the app's network/case option). + +[source: case file: data/case_ACTIVSg200.m | harvested: 2026-09-07 | auto-generated, review before trusting] +case_ACTIVSg200.m: 200 buses, 49 generators, 245 branches. MATPOWER-format network case usable with ExaGO OPF apps (pass via the app's network/case option). diff --git a/agentigrid/rag/corpus/exago_dcopflow_help.txt b/agentigrid/rag/corpus/exago_dcopflow_help.txt new file mode 100644 index 00000000..4c1b4ea7 --- /dev/null +++ b/agentigrid/rag/corpus/exago_dcopflow_help.txt @@ -0,0 +1,25 @@ +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +dcopflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `dcopflow `. See the option chunks below for flags. + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago dcopflow --help | binary: dcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/exago_exemplars_from_runs.txt b/agentigrid/rag/corpus/exago_exemplars_from_runs.txt new file mode 100644 index 00000000..bee690f0 --- /dev/null +++ b/agentigrid/rag/corpus/exago_exemplars_from_runs.txt @@ -0,0 +1,137 @@ +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Lower the total generation cost by at least 5% while serving the same load and without changing generator cost curves, staying feasible +Step applied: set_gen_status x13 on buses 435, 436, 264, 265, 275, 361, 362, 363, 388, 414, 442, 463, 494 (status=1) +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "set_gen_status x13 on buses 435, 436, 264, 265, 275, 361, 362, 363, 388, 414, 442, 463, 494 (status=1)", "commands": [{"action": "set_gen_status", "bus": 435, "status": 1}, {"action": "set_gen_status", "bus": 436, "status": 1}, {"action": "set_gen_status", "bus": 264, "status": 1}, {"action": "set_gen_status", "bus": 265, "status": 1}, {"action": "set_gen_status", "bus": 275, "status": 1}, {"action": "set_gen_status", "bus": 361, "status": 1}, {"action": "set_gen_status", "bus": 362, "status": 1}, {"action": "set_gen_status", "bus": 363, "status": 1}, {"action": "set_gen_status", "bus": 388, "status": 1}, {"action": "set_gen_status", "bus": 414, "status": 1}, {"action": "set_gen_status", "bus": 442, "status": 1}, {"action": "set_gen_status", "bus": 463, "status": 1}, {"action": "set_gen_status", "bus": 494, "status": 1}]} +Result: OPFLOW converged, feasible, 0 violations, objective 70,955.62 (-2.2% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Lower the total generation cost by at least 5% while serving the same load, without changing generator cost curves, and without relaxing bus voltage limits or line ratings, staying feasible +Step applied: set_gen_status x4 on buses 435, 436, 125, 126 (status=1) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_gen_status x4 on buses 435, 436, 125, 126 (status=1)", "commands": [{"action": "set_gen_status", "bus": 435, "status": 1}, {"action": "set_gen_status", "bus": 436, "status": 1}, {"action": "set_gen_status", "bus": 125, "status": 1}, {"action": "set_gen_status", "bus": 126, "status": 1}]} +Result: OPFLOW converged, feasible, 0 violations, objective 57,598.76 (-20.6% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=1.05) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=1.05)", "commands": [{"action": "scale_all_loads", "factor": 1.05}]} +Result: OPFLOW converged, feasible, 0 violations, objective 81,984.48 (+13.0% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=1.07) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=1.07)", "commands": [{"action": "scale_all_loads", "factor": 1.07}]} +Result: OPFLOW converged, feasible, 0 violations, objective 86,031.33 (+18.5% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=1.085) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=1.085)", "commands": [{"action": "scale_all_loads", "factor": 1.085}]} +Result: OPFLOW converged, feasible, 0 violations, objective 89,169.69 (+22.9% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=1.075) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=1.075)", "commands": [{"action": "scale_all_loads", "factor": 1.075}]} +Result: OPFLOW converged, feasible, 0 violations, objective 87,070.70 (+20.0% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 72,990.37 (+0.6% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load or raising line ratings +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 72,990.37 (+0.6% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: opflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load or raising line ratings +Step applied: set_shunt_susceptance(bus=130, Bs=20) +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "set_shunt_susceptance(bus=130, Bs=20)", "commands": [{"action": "set_shunt_susceptance", "bus": 130, "Bs": 20.0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 72,814.56 (+0.3% vs. base 72,578.30). + +[source: solver-certified AgentiGrid run | app: pflow | network: case_ACTIVSg500 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Relieve every overloaded line without shedding load, changing line ratings, or relaxing bus voltage limits +Step applied: set_all_bus_vlimits(Vmin=0.95, Vmax=1.05); set_gen_dispatch(bus=145, Pg=250.55) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.95, Vmax=1.05); set_gen_dispatch(bus=145, Pg=250.55)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.95, "Vmax": 1.05}, {"action": "set_gen_dispatch", "bus": 145, "Pg": 250.55}]} +Result: PFLOW converged, feasible, 0 violations, objective 72,781.49 (-0.0% vs. base 72,783.43). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Take the largest generator out of service and restore a feasible, low-cost operating point while serving the same load and without changing generator cost curves +Step applied: set_gen_status(bus=69, status=0) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_gen_status(bus=69, status=0)", "commands": [{"action": "set_gen_status", "bus": 69, "status": 0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 134,146.24 (+3.5% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Take the largest generator out of service and restore a feasible, low-cost operating point while serving the same load, without changing generator cost curves, and without relaxing bus voltage limits or line ratings +Step applied: set_gen_status(bus=69, status=0) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_gen_status(bus=69, status=0)", "commands": [{"action": "set_gen_status", "bus": 69, "status": 0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 134,146.24 (+3.5% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=2) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=2)", "commands": [{"action": "scale_all_loads", "factor": 2.0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 309,043.83 (+138.3% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Determine how much uniform load growth the system tolerates while remaining feasible, testing growth in steps +Step applied: scale_all_loads(factor=1.5) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "scale_all_loads(factor=1.5)", "commands": [{"action": "scale_all_loads", "factor": 1.5}]} +Result: OPFLOW converged, feasible, 0 violations, objective 216,185.54 (+66.7% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 129,839.86 (+0.1% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: opflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load or raising line ratings +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 129,839.86 (+0.1% vs. base 129,660.69). + +[source: solver-certified AgentiGrid run | app: pflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Bring every bus voltage within 0.95-1.05 pu without shedding load or changing the voltage limits +Step applied: set_all_bus_vlimits(Vmin=0.95, Vmax=1.05); set_gen_voltage(bus=76, Vg=1.02); set_shunt_susceptance x2 on buses 53, 118 +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.95, Vmax=1.05); set_gen_voltage(bus=76, Vg=1.02); set_shunt_susceptance x2 on buses 53, 118", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.95, "Vmax": 1.05}, {"action": "set_gen_voltage", "bus": 76, "Vg": 1.02}, {"action": "set_shunt_susceptance", "bus": 53, "Bs": 15.0}, {"action": "set_shunt_susceptance", "bus": 118, "Bs": 10.0}]} +Result: PFLOW converged, feasible, 0 violations, objective 131,234.89 (+0.0% vs. base 131,220.52). + +[source: solver-certified AgentiGrid run | app: scopflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Obtain a contingency-secure dispatch with all bus voltages held between 0.96 and 1.04 pu, without shedding load +Step applied: set_all_bus_vlimits(Vmin=0.96, Vmax=1.04) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.96, Vmax=1.04)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.96, "Vmax": 1.04}]} +Result: SCOPFLOW converged, feasible, 0 violations, objective 129,778.73 (+0.1% vs. base 129,660.73). + +[source: solver-certified AgentiGrid run | app: scopflow | network: case118 | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Obtain a contingency-secure dispatch with all bus voltages held between 0.96 and 1.04 pu, without shedding load or raising line ratings +Step applied: set_all_bus_vlimits(Vmin=0.96, Vmax=1.04) +Correct response (JSON): {"action": "modify", "mode": "accumulative", "description": "set_all_bus_vlimits(Vmin=0.96, Vmax=1.04)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.96, "Vmax": 1.04}]} +Result: SCOPFLOW converged, feasible, 0 violations, objective 129,778.73 (+0.1% vs. base 129,660.73). + +[source: solver-certified AgentiGrid run | app: opflow | network: case9mod | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Take the largest generator out of service and restore a feasible, low-cost operating point while serving the same load and without changing generator cost curves +Step applied: set_gen_status(bus=1, status=0) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_gen_status(bus=1, status=0)", "commands": [{"action": "set_gen_status", "bus": 1, "status": 0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 4,890.19 (+18.0% vs. base 4,144.46). + +[source: solver-certified AgentiGrid run | app: opflow | network: case9mod | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Take the largest generator out of service and restore a feasible, low-cost operating point while serving the same load, without changing generator cost curves, and without relaxing bus voltage limits or line ratings +Step applied: set_gen_status(bus=1, status=0) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_gen_status(bus=1, status=0)", "commands": [{"action": "set_gen_status", "bus": 1, "status": 0}]} +Result: OPFLOW converged, feasible, 0 violations, objective 4,890.19 (+18.0% vs. base 4,144.46). + +[source: solver-certified AgentiGrid run | app: opflow | network: case9mod | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 4,151.55 (+0.2% vs. base 4,144.46). + +[source: solver-certified AgentiGrid run | app: opflow | network: case9mod | generated by: claude-sonnet-5 | harvested: 2026-09-28] +Goal: Keep every bus voltage between 0.97 and 1.03 pu at the lowest cost you can find, without shedding load or raising line ratings +Step applied: set_all_bus_vlimits(Vmin=0.97, Vmax=1.03) +Correct response (JSON): {"action": "modify", "mode": "fresh", "description": "set_all_bus_vlimits(Vmin=0.97, Vmax=1.03)", "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]} +Result: OPFLOW converged, feasible, 0 violations, objective 4,151.55 (+0.2% vs. base 4,144.46). diff --git a/agentigrid/rag/corpus/exago_opflow_help.txt b/agentigrid/rag/corpus/exago_opflow_help.txt new file mode 100644 index 00000000..c6f8b7c3 --- /dev/null +++ b/agentigrid/rag/corpus/exago_opflow_help.txt @@ -0,0 +1,25 @@ +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +opflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `opflow `. See the option chunks below for flags. + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago opflow --help | binary: opflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/exago_pflow_help.txt b/agentigrid/rag/corpus/exago_pflow_help.txt new file mode 100644 index 00000000..d37fb063 --- /dev/null +++ b/agentigrid/rag/corpus/exago_pflow_help.txt @@ -0,0 +1,25 @@ +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +pflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `pflow `. See the option chunks below for flags. + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago pflow --help | binary: pflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/exago_scopflow_help.txt b/agentigrid/rag/corpus/exago_scopflow_help.txt new file mode 100644 index 00000000..99368c53 --- /dev/null +++ b/agentigrid/rag/corpus/exago_scopflow_help.txt @@ -0,0 +1,25 @@ +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +scopflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `scopflow `. See the option chunks below for flags. + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago scopflow --help | binary: scopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/exago_sopflow_help.txt b/agentigrid/rag/corpus/exago_sopflow_help.txt new file mode 100644 index 00000000..e4f461df --- /dev/null +++ b/agentigrid/rag/corpus/exago_sopflow_help.txt @@ -0,0 +1,25 @@ +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +sopflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `sopflow `. See the option chunks below for flags. + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago sopflow --help | binary: sopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/exago_tcopflow_help.txt b/agentigrid/rag/corpus/exago_tcopflow_help.txt new file mode 100644 index 00000000..df5245e3 --- /dev/null +++ b/agentigrid/rag/corpus/exago_tcopflow_help.txt @@ -0,0 +1,25 @@ +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +tcopflow: ExaGO application. Version ExaGO 2.0.0 built on Sep 5 2026. Invoked as `tcopflow `. See the option chunks below for flags. + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +ExaGO 2.0.0 built on Sep 5 2026 + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Arguments for %s: + -help + Print help message (type: flag) + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -version + Print version information (type: flag) + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -config + Print configuration options used to build ExaGO (type: flag) + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +[source: exago tcopflow --help | binary: tcopflow | version: ExaGO 2.0.0 built on Sep 5 2026 | harvested: 2026-09-07 | auto-generated, review before trusting] +Please enter a valid application name. diff --git a/agentigrid/rag/corpus/example_scopf_spec.txt b/agentigrid/rag/corpus/example_scopf_spec.txt new file mode 100644 index 00000000..ab97c0e9 --- /dev/null +++ b/agentigrid/rag/corpus/example_scopf_spec.txt @@ -0,0 +1,9 @@ +[source: hand-written methodology exemplar | example network: case118 (held out from the evaluation)] +Request: "Assess the security of case118 under single-element outages." +Correct specification: + application: scopflow + base_case: case118.m + contingencies: case118.cont (every contingency in the file, not a subset) + metric: post-contingency feasibility and cost + acceptance: feasible under all listed contingencies +Note: a SCOPFLOW run that selects only some of the contingencies, or none, still converges and returns a cost, but it does not answer an N-1 question. diff --git a/agentigrid/rag/corpus/n1_methodology.txt b/agentigrid/rag/corpus/n1_methodology.txt new file mode 100644 index 00000000..afa08b57 --- /dev/null +++ b/agentigrid/rag/corpus/n1_methodology.txt @@ -0,0 +1,3 @@ +N-1 security analysis checks whether the grid stays feasible after losing any single element. +It requires a contingency set. A base-case-only study does NOT answer an N-1 question. +In ExaGO, security-constrained OPF runs with SCOPFLOW using a contingency (.cont) file matching the base case. diff --git a/agentigrid/rag/corpus/opflow_help.txt b/agentigrid/rag/corpus/opflow_help.txt new file mode 100644 index 00000000..0b852396 --- /dev/null +++ b/agentigrid/rag/corpus/opflow_help.txt @@ -0,0 +1,16 @@ +ExaGO 2.0.0 built on Sep 5 2026 + +Arguments for %s: + -help + Print help message (type: flag) + + -version + Print version information (type: flag) + + -config + Print configuration options used to build ExaGO (type: flag) + + -options_file + Path to options file used to load additional ExaGO configuration options (type: flag) + +Please enter a valid application name. diff --git a/agentigrid/rag/tools/README.md b/agentigrid/rag/tools/README.md new file mode 100644 index 00000000..63ff0b7b --- /dev/null +++ b/agentigrid/rag/tools/README.md @@ -0,0 +1,68 @@ +# RAG corpus tools + +Scripts that build and check the AgentiGrid RAG knowledge base (`rag/corpus/`) +from **ground truth**: the ExaGO binaries, the case files and AgentiGrid's own +parser and validator, rather than hand-written prose. Every chunk carries a +`[source: ...]` tag, so a retrieved reference traces back to where it came from. + +Run everything **from the AgentiGrid root** (`agentigrid/`, where you launch +AgentiGrid). The defaults are relative paths: `applications/exago`, `data/exago` +and `rag/corpus`. + +## 1. Tool help and case metadata + + python rag/tools/rag_harvest.py + +This writes two kinds of file: + +- `rag/corpus/exago__help.txt`: one per application, from `--help` / `--version`; +- `rag/corpus/exago_cases.txt`: bus, generator and branch counts for each `.m` file. + +Options: + + --bin-dir applications/exago # where the binaries are + --data-dir data/exago # repeatable + --apps opflow scopflow ... # subset + --no-cases / --no-help + +## 2. Schema exemplars + + python rag/tools/rag_schema_exemplars.py # write rag/corpus/ + python rag/tools/rag_schema_exemplars.py --inspect # print, write nothing + +This writes `rag/corpus/agentigrid_schema_exemplars.txt`: complete, correct +responses, for example `set_all_bus_vlimits` nested inside a `modify` action and +never used as the top-level action. + +AgentiGrid's own parser, validator and modifier certify each exemplar on +`case118`. The exception is the phase-shifter exemplar, which no reference case +can host, so it is labelled "schema-validated only". + +## 3. Audit and freeze + + python rag/tools/corpus_guard.py rag/corpus # personal paths only + python rag/tools/corpus_guard.py --spec my_eval_spec.json rag/corpus # + hold-out audit + python rag/tools/corpus_guard.py --spec my_eval_spec.json rag/corpus --freeze + +The audit flags three things: + +- the text of an evaluated goal; +- a worked example on an evaluated network; +- an absolute personal path. + +`--freeze` writes `corpus_manifest.json` with the corpus SHA-256. It refuses while any findings remain. A run's journal records `rag_config`, so the corpus a run used can be traced. + +The shipped corpus (v1) was audited and frozen this way. The hold-out spec it was audited against belongs to a separate evaluation and is not part of this repository. `exago_exemplars_from_runs.txt` holds solver-certified worked examples, scraped from successful AgentiGrid runs on networks and goals outside that evaluation. The scraper and the batch runner that produced them are not included here. + +Copyright: do not paste paper or textbook text into the corpus. State facts in +your own words and cite them, or keep such notes in a local, gitignored corpus. + +## 4. Re-ingest after any corpus change + + rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus + +## Notes + +- **Idempotent.** Re-running overwrites only the generated files. Hand-written corpus files are left alone. +- **Offline.** None of these scripts calls an LLM or the network. Only `ingest` does, to get Ollama embeddings. +- **Review before trusting.** Generated chunks carry an "auto-generated, review before trusting" tag. diff --git a/agentigrid/rag/tools/corpus_guard.py b/agentigrid/rag/tools/corpus_guard.py new file mode 100644 index 00000000..616a5816 --- /dev/null +++ b/agentigrid/rag/tools/corpus_guard.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Corpus guard: hold-out leakage audit + freeze manifest for the RAG corpus. + +When retrieval is evaluated on specific (network, goal) tasks, a corpus that +contains worked solutions to those same tasks measures lookup of the answer +rather than grounding. This tool audits every corpus chunk against one or more +evaluation specs (JSON with "goals": [{"text": ...}] and "cases": +[{"path"|"network": ...}]) and flags: + + * goal-leak — a chunk contains the text of an evaluated goal; + * eval-exemplar — a worked example (proposal / correct response / correct + specification) that names an evaluated network; + * personal-path — an absolute home / Windows user path (machine-specific + noise, and it must not reach a public repo). + +Plain facts about an evaluated network (e.g. its bus count in the case metadata) +are allowed: they are documentation, not a solution. + +`--freeze` writes corpus_manifest.json (per-file SHA-256 + an overall corpus +hash) so every run can record exactly which corpus it was grounded on. + + python rag/tools/corpus_guard.py rag/corpus # personal paths only + python rag/tools/corpus_guard.py --spec my_eval_spec.json rag/corpus + python rag/tools/corpus_guard.py --spec my_eval_spec.json rag/corpus --freeze +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +CORPUS_SUFFIXES = (".txt", ".md") +MANIFEST_NAME = "corpus_manifest.json" +_EXEMPLAR_MARKERS = ("proposal", "correct response", "correct specification", + "correct action", "commands actually run") +_PERSONAL_PATH = re.compile(r"(/home/[A-Za-z0-9._-]+/|/Users/[A-Za-z0-9._-]+/|[A-Za-z]:\\Users\\|/mnt/c/Users/)") + + +def _norm(text: str) -> str: + return re.sub(r"\s+", " ", text.lower()).strip() + + +def load_holdout(spec_paths: list[Path]) -> dict: + """Evaluated goal texts and network stems from evaluation spec(s).""" + goals, networks = set(), set() + for sp in spec_paths: + spec = json.loads(Path(sp).read_text()) + for g in spec.get("goals", []): + if g.get("text"): + goals.add(_norm(g["text"])) + for c in spec.get("cases", []): + if c.get("path"): + networks.add(Path(c["path"]).stem.lower()) + if c.get("network"): + networks.add(str(c["network"]).lower()) + return {"goals": goals, "networks": networks} + + +def _network_pattern(stem: str) -> re.Pattern: + """Match a network name as a whole token (case39 must not match case39x/case390). + + 'case_ACTIVSg200' and the short form 'ACTIVSg200' both count.""" + names = {stem, stem.removeprefix("case_"), stem.removeprefix("case")} + names = {n for n in names if len(n) >= 4} + alt = "|".join(re.escape(n) for n in sorted(names, key=len, reverse=True)) + return re.compile(rf"(? list[str]: + """Paragraph-level chunks (blank-line separated), mirroring ingest's granularity.""" + return [c.strip() for c in re.split(r"\n\s*\n", text) if c.strip()] + + +def audit_text(text: str, holdout: dict) -> list[dict]: + findings = [] + net_pats = {n: _network_pattern(n) for n in holdout["networks"]} + for i, chunk in enumerate(split_chunks(text)): + low = _norm(chunk) + for g in holdout["goals"]: + if g and g in low: + findings.append({"chunk": i, "kind": "goal-leak", "detail": g}) + if any(m in low for m in _EXEMPLAR_MARKERS): + for n, pat in net_pats.items(): + if pat.search(chunk): + findings.append({"chunk": i, "kind": "eval-exemplar", "detail": n}) + m = _PERSONAL_PATH.search(chunk) + if m: + findings.append({"chunk": i, "kind": "personal-path", "detail": m.group(0)}) + return findings + + +def corpus_files(corpus_dir: Path) -> list[Path]: + from agentigrid.rag.corpus_hash import corpus_files as _cf + return _cf(corpus_dir) + + +def corpus_hash(corpus_dir: Path) -> tuple[str, dict[str, str]]: + """Overall SHA-256 over (relative path, content hash) pairs (shared definition).""" + from agentigrid.rag.corpus_hash import corpus_hash as _ch + return _ch(corpus_dir) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("corpus", nargs="?", default="rag/corpus") + ap.add_argument("--spec", action="append", default=[], + help="evaluation spec whose goals/networks are held out (repeatable)") + ap.add_argument("--freeze", action="store_true", + help="write corpus_manifest.json (refused if the audit finds problems)") + args = ap.parse_args(argv) + + corpus_dir = Path(args.corpus) + if not corpus_dir.is_dir(): + print(f"corpus dir not found: {corpus_dir}", file=sys.stderr) + return 2 + holdout = load_holdout([Path(s) for s in args.spec]) if args.spec else {"goals": set(), "networks": set()} + if not args.spec: + print("warning: no --spec given; only personal paths are checked", file=sys.stderr) + + total = 0 + for p in corpus_files(corpus_dir): + for f in audit_text(p.read_text(encoding="utf-8", errors="replace"), holdout): + total += 1 + print(f"{p.relative_to(corpus_dir)} chunk {f['chunk']:>3} {f['kind']:<14} {f['detail']}") + digest, per_file = corpus_hash(corpus_dir) + print(f"\n{len(per_file)} file(s); {total} finding(s); corpus sha256 {digest}") + + if args.freeze: + if total: + print("Refusing to freeze: resolve the findings first.", file=sys.stderr) + return 1 + (corpus_dir / MANIFEST_NAME).write_text(json.dumps({ + "corpus_sha256": digest, "files": per_file, + "held_out_specs": args.spec, "frozen_at": datetime.now().isoformat(), + }, indent=2) + "\n") + print(f"Frozen: {corpus_dir / MANIFEST_NAME}") + return 1 if total else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agentigrid/rag/tools/rag_harvest.py b/agentigrid/rag/tools/rag_harvest.py new file mode 100755 index 00000000..b13560b5 --- /dev/null +++ b/agentigrid/rag/tools/rag_harvest.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Tier-1 RAG corpus harvester for AgentiGrid. + +Auto-generates provenance-tagged corpus chunks from GROUND TRUTH — the ExaGO +binaries themselves and the MATPOWER case files — so nothing is hand-authored +or invented. Every chunk is prefixed with a [source] tag, so when the retriever +surfaces it the provenance travels with the text (the corpus stays auditable). + +Two harvesters: + * app --help / --version -> rag/corpus/exago__help.txt + * .m case-file metadata -> rag/corpus/exago_cases.txt + +Run from the AgentiGrid project root, then re-ingest: + python rag_tools/rag_harvest.py + rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus + +Nothing here calls the LLM or the network. Safe to run repeatedly (idempotent: +it overwrites its own output files, leaving hand-written corpus files alone). +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import os +import re +import subprocess +import sys +from pathlib import Path + +DEFAULT_APPS = ["opflow", "scopflow", "tcopflow", "sopflow", "dcopflow", "pflow"] + +# Files this script writes. Used so a re-run only clobbers its own output and +# never a curated / hand-written corpus file. +_GENERATED_PREFIX = "exago_" + +_TODAY = _dt.date.today().isoformat() + + +def _tag(src: str) -> str: + """Compact provenance tag placed at the head of every chunk.""" + return f"[source: {src} | harvested: {_TODAY} | auto-generated, review before trusting]" + + +def _paragraphs(text: str) -> list[str]: + """Split raw text into non-trivial paragraphs (blank-line separated).""" + blocks = re.split(r"\n\s*\n", text) + out = [] + for b in blocks: + b = b.rstrip() + # keep anything with at least a couple of word characters + if len(re.sub(r"\W", "", b)) >= 3: + out.append(b) + return out + + +# --------------------------------------------------------------------------- +# Binary discovery + --help harvest +# --------------------------------------------------------------------------- + +def _resolve_binary(bin_dir: Path, app: str) -> Path | None: + cand = bin_dir / app + if cand.exists(): + return cand.resolve() + return None + + +def _run(cmd: list[str], timeout: int) -> tuple[int, str]: + """Run a command, return (returncode, combined stdout+stderr). Never raises.""" + try: + p = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + return p.returncode, (p.stdout or "") + (p.stderr or "") + except FileNotFoundError: + return 127, "binary not found" + except subprocess.TimeoutExpired: + return 124, "timed out" + except Exception as e: # pragma: no cover - defensive + return 1, f"error: {e}" + + +def harvest_help(bin_dir: Path, out_dir: Path, apps: list[str], timeout: int) -> list[str]: + written = [] + for app in apps: + binary = _resolve_binary(bin_dir, app) + if binary is None: + print(f" [skip] {app}: no binary at {bin_dir/app}", file=sys.stderr) + continue + + # ExaGO help usually prints on --help; some options print to stderr. + rc, help_text = _run([str(binary), "--help"], timeout) + if not help_text.strip() or rc == 127: + print(f" [skip] {app}: --help produced nothing (rc={rc})", file=sys.stderr) + continue + + vrc, vtext = _run([str(binary), "--version"], timeout) + version = vtext.strip().splitlines()[0] if (vrc == 0 and vtext.strip()) else "unknown" + + # basename only: the absolute path is machine-specific and must not enter the corpus + src = f"exago {app} --help | binary: {Path(str(binary)).name} | version: {version}" + chunks = [f"{_tag(src)}\n{para}" for para in _paragraphs(help_text)] + if not chunks: + continue + + # A leading one-line summary chunk so a query like "how do I run scopflow" + # retrieves the app's identity even before matching a specific option. + summary = ( + f"{_tag(src)}\n{app}: ExaGO application. Version {version}. " + f"Invoked as `{app} `. See the option chunks below for flags." + ) + chunks.insert(0, summary) + + out_path = out_dir / f"{_GENERATED_PREFIX}{app}_help.txt" + out_path.write_text("\n\n".join(chunks) + "\n", encoding="utf-8") + written.append(out_path.name) + print(f" [ok] {app}: {len(chunks)} chunks -> {out_path.name}") + return written + + +# --------------------------------------------------------------------------- +# MATPOWER .m case-file metadata harvest +# --------------------------------------------------------------------------- + +def _count_matpower_block(text: str, name: str) -> int | None: + """Count data rows in `mpc. = [ ... ];`. Returns None if not present.""" + m = re.search(rf"mpc\.{name}\s*=\s*\[(.*?)\]\s*;", text, re.DOTALL) + if not m: + return None + rows = 0 + for line in m.group(1).splitlines(): + line = line.strip() + if not line or line.startswith("%"): + continue + if re.search(r"\d", line): + rows += 1 + return rows + + +def harvest_cases(data_dirs: list[Path], out_dir: Path) -> list[str]: + entries = [] + seen = set() + for d in data_dirs: + if not d.exists(): + continue + for m_file in sorted(d.rglob("*.m")): + if m_file.name in seen: + continue + try: + text = m_file.read_text(encoding="utf-8", errors="replace") + except Exception: + continue + buses = _count_matpower_block(text, "bus") + gens = _count_matpower_block(text, "gen") + branches = _count_matpower_block(text, "branch") + if buses is None and gens is None and branches is None: + continue # not a MATPOWER case + seen.add(m_file.name) + parts = [] + if buses is not None: + parts.append(f"{buses} buses") + if gens is not None: + parts.append(f"{gens} generators") + if branches is not None: + parts.append(f"{branches} branches") + src = f"case file: {m_file}" + entries.append( + f"{_tag(src)}\n" + f"{m_file.name}: {', '.join(parts)}. " + f"MATPOWER-format network case usable with ExaGO OPF apps " + f"(pass via the app's network/case option)." + ) + if not entries: + print(" [skip] cases: no MATPOWER .m files found", file=sys.stderr) + return [] + out_path = out_dir / f"{_GENERATED_PREFIX}cases.txt" + out_path.write_text("\n\n".join(entries) + "\n", encoding="utf-8") + print(f" [ok] cases: {len(entries)} case files -> {out_path.name}") + return [out_path.name] + + +# --------------------------------------------------------------------------- + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--bin-dir", default="applications/exago", + help="Directory holding the ExaGO app binaries/symlinks (default: ./applications/exago)") + ap.add_argument("--data-dir", action="append", default=None, + help="Directory to scan for .m case files (repeatable). " + "Default: ./data/exago/datafiles and ./data/exago/examples") + ap.add_argument("--out", default="rag/corpus", + help="Corpus output directory (default: rag/corpus)") + ap.add_argument("--apps", nargs="*", default=DEFAULT_APPS, + help=f"Apps to harvest help from (default: {' '.join(DEFAULT_APPS)})") + ap.add_argument("--timeout", type=int, default=20, help="Per-command timeout seconds") + ap.add_argument("--no-help", action="store_true", help="Skip the --help harvest") + ap.add_argument("--no-cases", action="store_true", help="Skip the case-file harvest") + args = ap.parse_args() + + bin_dir = Path(args.bin_dir) + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + + data_dirs = [Path(p) for p in (args.data_dir or ["data/exago/datafiles", "data/exago/examples"])] + + print(f"Harvesting into {out_dir}/ (provenance-tagged, safe to re-run)") + written = [] + if not args.no_help: + # Fallback: if ./applications is empty, hint at the build dir. + if not any(bin_dir.glob("*")): + print(f" note: {bin_dir}/ looks empty. Point --bin-dir at your " + f"ExaGO build/applications, or run from the project root.", + file=sys.stderr) + written += harvest_help(bin_dir, out_dir, args.apps, args.timeout) + if not args.no_cases: + written += harvest_cases(data_dirs, out_dir) + + if not written: + print("Nothing harvested. Check --bin-dir / --data-dir.", file=sys.stderr) + return 1 + print(f"\nDone. {len(written)} corpus file(s) written. Re-ingest with:") + print(" rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agentigrid/rag/tools/rag_schema_exemplars.py b/agentigrid/rag/tools/rag_schema_exemplars.py new file mode 100644 index 00000000..e1edfa2e --- /dev/null +++ b/agentigrid/rag/tools/rag_schema_exemplars.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Schema exemplars — the AgentiGrid action/command structure as worked examples. + +Many invalid proposals are API-structure mistakes, not power-systems mistakes: +e.g. emitting {"action": "set_all_bus_vlimits", ...} as the TOP-LEVEL action +instead of nesting it inside a "modify" action's "commands" list. The command +list is in the system prompt, but only as bare command objects; these exemplars +show complete, correct responses. + +Every exemplar is CERTIFIED by AgentiGrid's own code before it is written: + 1. the top-level action is one the agent loop dispatches; + 2. every nested command passes `parse_command` (the loop's parser); + 3. where a held-out reference case can host it, the commands pass the + validator and are applied by `apply_modifications` with nothing skipped. +Exemplars that cannot be applied to a held-out case (e.g. phase-shifter commands, +since no held-out case has a phase shifter) are labelled "schema-validated only". + +Reference case: case118 (kept out of the evaluation tasks the frozen corpus was +audited against). Bus and branch numbers therefore refer to case118; the +exemplars teach structure. + + python rag/tools/rag_schema_exemplars.py # write rag/corpus/ + python rag/tools/rag_schema_exemplars.py --inspect # print, write nothing +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +OUT_NAME = "agentigrid_schema_exemplars.txt" +DEFAULT_REF_CASE = "../datafiles/case118.m" +# Top-level actions dispatched by AgentiGrid's agent loop (agent_loop._run_iteration). +TOP_LEVEL_ACTIONS = ("modify", "complete", "analyze", "sweep", "explore", "select") + +# Each exemplar: request, apps it applies to, the response the agent should emit, +# and an optional note. Requests are generic single-step instructions, never the +# evaluated goals (corpus_guard.py audits this against an evaluation spec). +EXEMPLARS: list[dict] = [ + {"request": "Set voltage limits on every bus to 0.97-1.03 pu.", + "apps": ["opflow", "scopflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "All-bus voltage band 0.97-1.03 pu", + "reasoning": "A system-wide band is one set_all_bus_vlimits command, nested in modify.", + "commands": [{"action": "set_all_bus_vlimits", "Vmin": 0.97, "Vmax": 1.03}]}, + "note": "set_all_bus_vlimits is a COMMAND, not an action. Wrong: " + "{\"action\": \"set_all_bus_vlimits\", \"Vmin\": 0.97, \"Vmax\": 1.03} as the whole " + "response. Right: nest it in the \"commands\" list of a \"modify\" action."}, + {"request": "Tighten the voltage limits of bus 12 only to 0.98-1.02 pu.", + "apps": ["opflow", "scopflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Bus 12 voltage band 0.98-1.02 pu", + "reasoning": "Single-bus limits use set_bus_vlimits.", + "commands": [{"action": "set_bus_vlimits", "bus": 12, "Vmin": 0.98, "Vmax": 1.02}]}, + "note": "In OPFLOW/SCOPFLOW, bus voltages are decision variables: constrain them with " + "set_bus_vlimits / set_all_bus_vlimits, not with set_gen_voltage."}, + {"request": "Increase all loads in the network by 8%.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "fresh", + "description": "Uniform load scaling x1.08", + "reasoning": "Uniform scaling of every load is scale_all_loads with factor 1.08.", + "commands": [{"action": "scale_all_loads", "factor": 1.08}]}, + "note": "factor is a multiplier: 1.08 = +8%, 0.9 = -10%. mode \"fresh\" applies the change to " + "the base case; \"accumulative\" stacks it on the current state."}, + {"request": "Set the load at bus 15 to 95 MW and 30 MVAr.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Bus 15 load 95 MW / 30 MVAr", + "reasoning": "Absolute load values use set_load.", + "commands": [{"action": "set_load", "bus": 15, "Pd": 95.0, "Qd": 30.0}]}}, + {"request": "Add a new 40 MW load at bus 20 on top of its existing load.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Add 40 MW at bus 20", + "reasoning": "add_load_at_bus adds to the existing load; set_load would replace it.", + "commands": [{"action": "add_load_at_bus", "bus": 20, "Pd": 40.0, "Qd": 0.0}]}}, + {"request": "Take the generator at bus 10 out of service.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "fresh", + "description": "Generator at bus 10 offline", + "reasoning": "Generator outage is set_gen_status with status 0.", + "commands": [{"action": "set_gen_status", "bus": 10, "status": 0}]}}, + {"request": "Set the output of the generator at bus 12 to 150 MW.", + "apps": ["pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Bus 12 generator Pg = 150 MW", + "reasoning": "In PFLOW the dispatch is fixed by set_gen_dispatch; Pg must lie in [Pmin, Pmax].", + "commands": [{"action": "set_gen_dispatch", "bus": 12, "Pg": 150.0}]}, + "note": "In OPFLOW/SCOPFLOW the optimizer re-dispatches generators; to force an output there, " + "bound it instead (this command sets Pg as the starting value)."}, + {"request": "Raise the voltage setpoint of the generator at bus 10 to 1.04 pu.", + "apps": ["pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Bus 10 Vg = 1.04 pu", + "reasoning": "In PFLOW the generator voltage setpoint is enforced, so it is the primary voltage control.", + "commands": [{"action": "set_gen_voltage", "bus": 10, "Vg": 1.04}]}, + "note": "Only meaningful in PFLOW. In OPFLOW/SCOPFLOW the solver overrides Vg."}, + {"request": "Add 25 MVAr of capacitive shunt support at bus 44.", + "apps": ["pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Bus 44 shunt Bs = 25", + "reasoning": "Positive Bs is capacitive and raises the local voltage.", + "commands": [{"action": "set_shunt_susceptance", "bus": 44, "Bs": 25.0}]}}, + {"request": "Change the tap ratio of the transformer between buses 8 and 5 to 1.0.", + "apps": ["pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Transformer 8-5 tap = 1.0", + "reasoning": "Tap changes use set_tap_ratio on a transformer branch.", + "commands": [{"action": "set_tap_ratio", "fbus": 8, "tbus": 5, "ratio": 1.0}]}, + "note": "Only transformer branches (ratio != 0 in the case) accept a tap change."}, + {"request": "Set the phase shifter between buses 30 and 17 to 4 degrees.", + "apps": ["pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Phase shift 30-17 = 4 deg", + "reasoning": "Phase-shifter angle changes use set_phase_shift_angle.", + "commands": [{"action": "set_phase_shift_angle", "fbus": 30, "tbus": 17, "angle": 4.0}]}, + "note": "Only branches that are phase shifters (angle != 0 in the case) accept this command."}, + {"request": "Take the line between buses 23 and 24 out of service.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "fresh", + "description": "Line 23-24 out of service", + "reasoning": "A branch outage is set_branch_status with status 0.", + "commands": [{"action": "set_branch_status", "fbus": 23, "tbus": 24, "status": 0}]}, + "note": "In SCOPFLOW this permanently changes the topology; contingencies come from the " + "contingency file, not from set_branch_status."}, + {"request": "Raise the thermal rating of the line between buses 30 and 38 to 250 MVA.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "modify", "mode": "accumulative", + "description": "Line 30-38 rateA = 250 MVA", + "reasoning": "Branch ratings use set_branch_rate (rateA in MVA).", + "commands": [{"action": "set_branch_rate", "fbus": 30, "tbus": 38, "rateA": 250.0}]}}, + {"request": "Scale all loads by 1.05 and set the voltage band on every bus to 0.96-1.04 pu in one step.", + "apps": ["opflow", "scopflow"], + "response": {"action": "modify", "mode": "fresh", + "description": "Loads x1.05 with 0.96-1.04 pu band", + "reasoning": "Several commands go in one commands list, applied in order.", + "commands": [{"action": "scale_all_loads", "factor": 1.05}, + {"action": "set_all_bus_vlimits", "Vmin": 0.96, "Vmax": 1.04}]}}, + {"request": "List the 3 buses nearest to bus 49 before deciding.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "analyze", "query_type": "nearest_neighbors", "bus": 49, "k": 3, + "reasoning": "Topology questions use a structured query_type, not free text."}}, + {"request": "The target is met; stop and report.", + "apps": ["opflow", "scopflow", "pflow"], + "response": {"action": "complete", + "reasoning": "The last feasible solve satisfies the stated target.", + "findings": {"summary": "One-sentence answer to the goal, with the key number.", + "details": "Iteration, objective value and constraint status supporting it."}}, + "note": "complete carries findings; it does not carry commands."}, +] + + +def certify(ex: dict, ref_net) -> str: + """Run AgentiGrid's own parser/validator/modifier on an exemplar. + + Returns the certification label; raises ValueError if the exemplar is invalid. + """ + from agentigrid.exago_engine.commands import parse_command + from agentigrid.exago_engine.modifier import apply_modifications + + resp = ex["response"] + action = resp.get("action") + if action not in TOP_LEVEL_ACTIONS: + raise ValueError(f"top-level action {action!r} is not dispatched by the agent loop") + raws = resp.get("commands", []) + if action != "modify": + if raws: + raise ValueError(f"{action!r} must not carry commands") + return "structure-validated" + if not raws: + raise ValueError("modify with no commands") + cmds = [parse_command(r) for r in raws] # raises ValueError on a bad command + if ref_net is None: + return "schema-validated only" + apps = ex.get("apps", []) + app = "pflow" if apps == ["pflow"] else "opflow" + _net, report = apply_modifications(ref_net, cmds, application=app) + if report.skipped: + reasons = "; ".join("; ".join(r) for _c, r in report.skipped) + reasons = reasons.split(".")[0][:90] + return f"schema-validated only (not applicable to reference case: {reasons})" + return "validated: parsed and applied on held-out case118" + + +def render(ex: dict, label: str) -> str: + tag = (f"[source: schema exemplar | generated from AgentiGrid's command set | {label} | " + f"apps: {', '.join(ex.get('apps', []))}]") + lines = [tag, f"Request: {ex['request']}", + f"Correct response (JSON): {json.dumps(ex['response'], separators=(', ', ': '))}"] + if ex.get("note"): + lines.append(f"Note: {ex['note']}") + return "\n".join(lines) + + +def build(ref_case: Path | None) -> list[str]: + ref_net = None + if ref_case is not None and ref_case.exists(): + from agentigrid.exago_parsers.matpower_parser import parse_matpower + ref_net = parse_matpower(ref_case) + return [render(ex, certify(ex, ref_net)) for ex in EXEMPLARS] + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--ref-case", default=DEFAULT_REF_CASE, + help="held-out MATPOWER case used to certify applicability (default: case118)") + ap.add_argument("--out", default="rag/corpus") + ap.add_argument("--inspect", action="store_true") + args = ap.parse_args(argv) + + ref = Path(args.ref_case) + if not ref.exists(): + print(f"warning: reference case {ref} not found; exemplars will be schema-validated only", + file=sys.stderr) + chunks = build(ref) + if args.inspect: + print("\n\n".join(chunks)) + return 0 + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + (out / OUT_NAME).write_text("\n\n".join(chunks) + "\n", encoding="utf-8") + print(f"Wrote {len(chunks)} schema exemplar(s) -> {out / OUT_NAME}") + print("Re-ingest with: rm -rf rag/store && python -m agentigrid.rag.ingest rag/corpus") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agentigrid/requirements-rag.txt b/agentigrid/requirements-rag.txt new file mode 100644 index 00000000..6ae55949 --- /dev/null +++ b/agentigrid/requirements-rag.txt @@ -0,0 +1,9 @@ +# Optional dependencies for retrieval grounding (RAG). Not needed when RAG is off +# (the default). Install with: pip install -r requirements-rag.txt +# Also needs a running Ollama server with the embedding model: +# ollama pull nomic-embed-text +chromadb>=0.4 +requests>=2 + +# Optional: local cross-encoder grader for corrective mode (AGENTIGRID_CRAG_GRADER=reranker) +# sentence-transformers>=3 diff --git a/agentigrid/tests/README.md b/agentigrid/tests/README.md index 332826d1..0a8e7e81 100644 --- a/agentigrid/tests/README.md +++ b/agentigrid/tests/README.md @@ -15,6 +15,7 @@ Tests are grouped by the part of AgentiGrid they cover: | `journal/` | Search journal, objectives, Pareto front, session save and report regeneration | | `analyses/` | Explore, sweeps, boundary search, contingency, relief, reserve and topology | | `e2e/` | End-to-end runs through the agent loop | +| `rag/` | Optional retrieval grounding: retrievers, corrective-RAG graders, corpus tools (`agentigrid/rag/`, `rag/tools/`) | | `fixtures/` | Shared sample files (e.g. `sample_opflow_output.txt`) | Run the fast tests from the AgentiGrid root: diff --git a/agentigrid/tests/rag/__init__.py b/agentigrid/tests/rag/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/agentigrid/tests/rag/test_agent_loop_rag.py b/agentigrid/tests/rag/test_agent_loop_rag.py new file mode 100644 index 00000000..976ea16d --- /dev/null +++ b/agentigrid/tests/rag/test_agent_loop_rag.py @@ -0,0 +1,119 @@ +"""Retrieval in the agent loop: off by default, generation-stage only, journaled.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from agentigrid.exago_engine.agent_loop import AgentLoopController +from tests.agent_loop.test_agent_loop import ( + BASE_CASE, MockBackend, _has_base_case, _has_sample, _make_config, _make_llm_response, + _make_sim_result, _objectives_response, _sample_stdout, +) + +pytestmark = [ + pytest.mark.skipif(not _has_base_case, reason="Base case .m file not found"), + pytest.mark.skipif(not _has_sample, reason="sample_opflow_output.txt not found"), +] + +_COMPLETE = {"action": "complete", "reasoning": "Done", "findings": {"summary": "ok"}} +_REF = "[ref 1 | score 0.80] [source: test] Use set_all_bus_vlimits inside a modify action." + + +class _FakeRetriever: + enabled = True + + def __init__(self): + self.queries: list[str] = [] + + def retrieve(self, query, k=None): + self.queries.append(query) + return _REF + + def describe(self): + return {"mode": "basic", "enabled": True, "k": 3, "min_score": 0.35, + "stats": {"calls": len(self.queries), "errors": 0, "empty": 0}} + + +def _run(tmp_path, monkeypatch, retriever=None, mode=None): + if mode is None: + monkeypatch.delenv("AGENTIGRID_RAG_MODE", raising=False) + monkeypatch.delenv("AGENTIGRID_RAG", raising=False) + else: + monkeypatch.setenv("AGENTIGRID_RAG_MODE", mode) + cfg = _make_config(tmp_path, max_iterations=3) + backend = MockBackend([_objectives_response(), _make_llm_response(_COMPLETE)]) + phases: list[str] = [] + patches = [ + patch("agentigrid.exago_engine.agent_loop.create_backend", return_value=backend), + patch("agentigrid.exago_engine.agent_loop.SimulationExecutor"), + ] + if retriever is not None: + patches.append(patch("agentigrid.exago_engine.agent_loop.build_retriever", return_value=retriever)) + with patches[0], patches[1] as mock_exec_cls, (patches[2] if len(patches) > 2 else _nullcontext()): + mock_executor = MagicMock() + mock_executor.run.return_value = _make_sim_result(stdout=_sample_stdout(), success=True) + mock_exec_cls.return_value = mock_executor + session = AgentLoopController(cfg, on_phase=lambda it, ph: phases.append(ph)).run(BASE_CASE, "Lower the cost") + return session, backend, phases + + +class _nullcontext: + def __enter__(self): + return None + + def __exit__(self, *exc): + return False + + +def _iteration_prompts(backend): + """User prompts of the per-iteration calls: the objective-extraction call comes + first and the post-search goal classification last; this run has one iteration.""" + return [user for _, user in backend.calls[1:-1]] + + +def test_rag_is_off_by_default(tmp_path, monkeypatch): + session, backend, phases = _run(tmp_path, monkeypatch) + assert all("Reference Material" not in p for p in _iteration_prompts(backend)) + assert not any(ph.startswith("rag_retrieved:") for ph in phases) + assert "llm_request" in phases + assert session.journal.rag_enabled is False + assert session.journal.rag_config["mode_env"] == "off" + + +def test_retrieved_references_reach_the_prompt_and_journal(tmp_path, monkeypatch): + fake = _FakeRetriever() + session, backend, phases = _run(tmp_path, monkeypatch, retriever=fake, mode="basic") + prompts = _iteration_prompts(backend) + assert len(prompts) == 1 + assert prompts[0].startswith("=== Section B: Reference Material (retrieved) ===") + assert _REF in prompts[0] + assert "Reference Material" not in backend.calls[-1][1] # post-search analysis is not grounded + assert fake.queries and set(fake.queries) == {"Lower the cost"} # the query is the goal text + i = phases.index("rag_retrieved:1:0.80") + assert phases[i + 1] == "llm_request" + assert session.journal.rag_enabled is True + cfg = session.journal.rag_config + assert cfg["mode"] == "basic" and cfg["mode_env"] == "basic" and cfg["stats"]["calls"] == len(fake.queries) + + +def test_failed_retrieval_leaves_the_prompt_unchanged(tmp_path, monkeypatch): + class _Empty(_FakeRetriever): + def retrieve(self, query, k=None): + self.queries.append(query) + return "" + + session, backend, phases = _run(tmp_path, monkeypatch, retriever=_Empty(), mode="basic") + assert all("Reference Material" not in p for p in _iteration_prompts(backend)) + assert "rag_retrieved:0:0.00" in phases + + +def test_journal_file_records_rag_fields(tmp_path, monkeypatch): + session, _, _ = _run(tmp_path, monkeypatch, retriever=_FakeRetriever(), mode="basic") + out = tmp_path / "j.json" + session.journal.export_json(out) + data = json.loads(out.read_text()) + assert data["rag_enabled"] is True and data["rag_config"]["mode"] == "basic" diff --git a/agentigrid/tests/rag/test_corpus_tools.py b/agentigrid/tests/rag/test_corpus_tools.py new file mode 100644 index 00000000..de0bb314 --- /dev/null +++ b/agentigrid/tests/rag/test_corpus_tools.py @@ -0,0 +1,114 @@ +"""Corpus tools: schema exemplars, hold-out / personal-path guard, frozen-corpus identity.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +CORPUS = ROOT / "rag" / "corpus" + + +def _load(name): + spec = importlib.util.spec_from_file_location(name, ROOT / "rag" / "tools" / f"{name}.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +schema = _load("rag_schema_exemplars") +guard = _load("corpus_guard") + + +# --- schema exemplars ----------------------------------------------------------- + +def test_schema_exemplars_certify_on_case118(): + chunks = schema.build(ROOT.parent / "datafiles" / "case118.m") + assert len(chunks) == len(schema.EXEMPLARS) + applied = sum("applied on held-out case118" in c for c in chunks) + assert applied >= 12 + # the one command without a host branch in any held-out case is labelled honestly + assert any("set_phase_shift_angle" in c and "schema-validated only" in c for c in chunks) + for ex in schema.EXEMPLARS: + assert ex["response"]["action"] in schema.TOP_LEVEL_ACTIONS + + +def test_schema_rejects_top_level_command(): + bad = {"request": "x", "apps": ["opflow"], + "response": {"action": "set_all_bus_vlimits", "Vmin": 0.9, "Vmax": 1.1}} + with pytest.raises(ValueError): + schema.certify(bad, None) + + +# --- guard ------------------------------------------------------------------------ + +def test_guard_flags(): + hold = {"goals": {guard._norm("Reduce total generation cost by 10%")}, + "networks": {"case_activsg200", "case39"}} + text = ("Request: Reduce total generation cost by 10%\n\n" + "Correct specification: application scopflow on case_ACTIVSg200\n\n" + "Case facts: case_ACTIVSg200 has 200 buses\n\n" + "Proposal ran /home/someone/x.m\n\n" + "Correct response on case390 is fine") + kinds = [f["kind"] for f in guard.audit_text(text, hold)] + assert kinds.count("goal-leak") == 1 + assert kinds.count("eval-exemplar") == 1 # facts-only chunk and case390 are not flagged + assert kinds.count("personal-path") == 1 + + +def test_freeze_refuses_on_findings_and_hashes(tmp_path): + (tmp_path / "a.txt").write_text("clean fact\n") + assert guard.main([str(tmp_path), "--freeze"]) == 0 + m = json.loads((tmp_path / guard.MANIFEST_NAME).read_text()) + assert len(m["corpus_sha256"]) == 64 and list(m["files"]) == ["a.txt"] + (tmp_path / "b.txt").write_text("path /home/someone/x\n") + assert guard.main([str(tmp_path), "--freeze"]) == 1 + + +def test_journal_roundtrips_skipped_commands(tmp_path): + from agentigrid.exago_engine.journal import JournalEntry, SearchJournal + j = SearchJournal() + e = JournalEntry(iteration=1, description="d", commands=[], objective_value=1.0, feasible=True, + convergence_status="CONVERGED", violations_count=0, voltage_min=1.0, voltage_max=1.0, + max_line_loading_pct=0.0, total_gen_mw=0.0, total_load_mw=0.0, llm_reasoning="", + mode="fresh", elapsed_seconds=0.0) + assert e.skipped_commands is None + e.skipped_commands = ["Skipped x"] + j.add_entry(e) + out = tmp_path / "j.json" + j.export_json(out) + assert json.loads(out.read_text())["entries"][0]["skipped_commands"] == ["Skipped x"] + + +# --- frozen-corpus identity ----------------------------------------------------- + +def test_corpus_status_tracks_freeze_and_ingest(tmp_path): + from agentigrid.rag.corpus_hash import CHUNKING, INGEST_MANIFEST_NAME, corpus_hash, corpus_status + corpus, store = tmp_path / "corpus", tmp_path / "store" + corpus.mkdir(); store.mkdir() + (corpus / "a.txt").write_text("alpha") + assert "not frozen" in corpus_status(corpus, store)["reason"] + assert guard.main([str(corpus), "--freeze"]) == 0 + assert "store not built" in corpus_status(corpus, store)["reason"] + digest, _ = corpus_hash(corpus) + (store / INGEST_MANIFEST_NAME).write_text(json.dumps({"corpus_sha256": digest, "chunking": dict(CHUNKING)})) + st = corpus_status(corpus, store) + assert st["ok"] and st["corpus_sha256"] == digest + (corpus / "a.txt").write_text("alpha changed") + assert "changed since it was frozen" in corpus_status(corpus, store)["reason"] + + + +def test_shipped_corpus_matches_its_manifest_and_is_clean(): + """The committed corpus is exactly the frozen one, with no personal paths.""" + from agentigrid.rag.corpus_hash import MANIFEST_NAME, corpus_hash + manifest = json.loads((CORPUS / MANIFEST_NAME).read_text()) + digest, per_file = corpus_hash(CORPUS) + assert digest == manifest["corpus_sha256"] + assert per_file == manifest["files"] + no_holdout = {"goals": set(), "networks": set()} + for f in guard.corpus_files(CORPUS): + assert guard.audit_text(f.read_text(encoding="utf-8"), no_holdout) == [], f.name diff --git a/agentigrid/tests/rag/test_grader_reranker.py b/agentigrid/tests/rag/test_grader_reranker.py new file mode 100644 index 00000000..3c7b9658 --- /dev/null +++ b/agentigrid/tests/rag/test_grader_reranker.py @@ -0,0 +1,115 @@ +"""Local cross-encoder grader for corrective RAG: seam contract, score mapping, failure +accounting, env wiring and journaled telemetry. Uses a fake model, so no +download or sentence-transformers install is needed.""" + +from __future__ import annotations + +import pytest + +from agentigrid.rag import build_retriever, resolve_crag_grader, _tau_overrides +from agentigrid.rag.corrective import CorrectiveRetriever +from agentigrid.rag.grader_reranker import RerankerGrader, to_unit_interval + +HITS = [("chunk about voltage limits", {}, 0.41), ("chunk about generator cost", {}, 0.33)] + + +class FakeCE: + def __init__(self, scores): + self.scores = scores + self.seen = None + + def predict(self, pairs, batch_size=16, show_progress_bar=False): + self.seen = pairs + return self.scores + + +def test_scores_pass_through_probabilities_and_sigmoid_logits(): + assert to_unit_interval([0.2, 0.9]) == [0.2, 0.9] + out = to_unit_interval([3.0, -3.0]) + assert out[0] > 0.95 and out[1] < 0.05 + + +def test_grader_scores_query_chunk_pairs(): + fake = FakeCE([0.8, 0.1]) + g = RerankerGrader(model_name="fake", _model=fake) + assert g("set voltage band", HITS) == [0.8, 0.1] + assert fake.seen == [("set voltage band", HITS[0][0]), ("set voltage band", HITS[1][0])] + d = g.describe() + assert d["grader"] == "reranker" and d["calls"] == 1 and d["failures"] == 0 + + +def test_unavailable_model_is_counted_and_reported(monkeypatch): + g = RerankerGrader(model_name="/nonexistent/model") + monkeypatch.setattr(g, "_ensure_model", lambda: None) + g._load_error = "OSError: not found" + with pytest.raises(RuntimeError): + g("q", HITS) + assert g.failures == 1 and "fallback" in g.describe()["grader"] + + +class _Base: + enabled, k, min_score = True, 3, 0.35 + + def query_hits(self, q, k): + return HITS + + def embed_text(self, t): + return [1.0, 0.0] + + +def test_corrective_counts_fallbacks_and_verdicts(): + class Broken: + def __call__(self, q, h): + raise RuntimeError("boom") + + def describe(self): + return {"grader": "reranker (fallback: cosine)"} + + cr = CorrectiveRetriever(_Base(), grader=Broken()) + cr.retrieve("voltage limits") + d = cr.describe() + assert d["stats"]["grader_fallbacks"] >= 1 + assert d["stats"]["calls"] == 1 + assert d["grader"] == "reranker (fallback: cosine)" + assert sum(d["stats"][v] for v in ("correct", "ambiguous", "incorrect")) >= 1 + + +def test_corrective_uses_reranker_scores_for_verdict(): + cr = CorrectiveRetriever(_Base(), grader=RerankerGrader(_model=FakeCE([0.9, 0.2]))) + cr.retrieve("voltage limits") + assert cr.stats["correct"] == 1 and cr.stats["grader_fallbacks"] == 0 + + +def test_env_selects_reranker_and_thresholds(monkeypatch): + monkeypatch.setenv("AGENTIGRID_RAG_MODE", "corrective") + monkeypatch.setenv("AGENTIGRID_CRAG_GRADER", "reranker") + monkeypatch.setenv("AGENTIGRID_CRAG_TAU_LOWER", "0.2") + monkeypatch.setenv("AGENTIGRID_CRAG_TAU_UPPER", "0.7") + assert resolve_crag_grader() == "reranker" + r = build_retriever(host="http://127.0.0.1:1") # store may be unavailable; config still applies + assert isinstance(r, CorrectiveRetriever) + assert (r.tau_lower, r.tau_upper) == (0.2, 0.7) + assert isinstance(r._grader, RerankerGrader) + + +def test_tau_overrides_validate(monkeypatch): + monkeypatch.setenv("AGENTIGRID_CRAG_TAU_LOWER", "0.8") + monkeypatch.setenv("AGENTIGRID_CRAG_TAU_UPPER", "0.2") + with pytest.raises(ValueError): + _tau_overrides() + + +def test_journal_carries_rag_config(tmp_path): + import json + from agentigrid.exago_engine.journal import SearchJournal + + j = SearchJournal() + j.rag_enabled = True + j.rag_config = {"mode": "corrective", "mode_env": "corrective", "grader": "reranker", + "stats": {"grader_fallbacks": 0, "correct": 3, "ambiguous": 1, "incorrect": 0, "withheld": 0}} + out = tmp_path / "j.json" + j.export_json(out) + data = json.loads(out.read_text()) + assert data["rag_enabled"] is True + assert data["rag_config"]["grader"] == "reranker" + assert data["rag_config"]["stats"]["correct"] == 3 diff --git a/agentigrid/tests/rag/test_retrieval_failures.py b/agentigrid/tests/rag/test_retrieval_failures.py new file mode 100644 index 00000000..3db6bb2c --- /dev/null +++ b/agentigrid/tests/rag/test_retrieval_failures.py @@ -0,0 +1,55 @@ +"""Retrieval failures are counted (not hidden), and the index identity is checked.""" + +from __future__ import annotations + +import json + +from agentigrid.rag.corrective import CorrectiveRetriever +from agentigrid.rag.retriever import Retriever + +class _DownStore: + def query(self, q, k): + raise ConnectionError("embedding server unreachable") + + +def _retriever(): + r = Retriever(enabled=False) + r.enabled, r._store = True, _DownStore() + return r + + +def test_basic_retriever_counts_errors(): + r = _retriever() + assert r.retrieve("reduce cost") == "" + assert r.stats == {"calls": 1, "errors": 1, "empty": 0} + assert r.describe()["stats"]["errors"] == 1 + + +def test_corrective_exposes_base_errors(): + c = CorrectiveRetriever(_retriever()) + assert c.retrieve("reduce cost") == "" + assert c.describe()["stats"]["retrieval_errors"] >= 1 + + +def test_chunking_keeps_worked_examples_whole(): + from agentigrid.rag.ingest import chunk + example = "Goal: lower cost\nStep applied: x\nCorrect response (JSON): " + '{"a": 1}, ' * 180 + "\nResult: ok" + assert 800 < len(example) < 3000 + assert chunk(example + "\n\nsecond paragraph") == [example, "second paragraph"] + long = "y" * 7000 + parts = chunk(long) + assert len(parts) == 3 and all(len(p) <= 3000 for p in parts) + + +def test_status_refuses_store_with_other_chunking(tmp_path): + from agentigrid.rag.corpus_hash import CHUNKING, INGEST_MANIFEST_NAME, MANIFEST_NAME, corpus_hash, corpus_status + corpus, store = tmp_path / "corpus", tmp_path / "store" + corpus.mkdir(); store.mkdir() + (corpus / "a.txt").write_text("hello") + digest, _ = corpus_hash(corpus) + (corpus / MANIFEST_NAME).write_text(json.dumps({"corpus_sha256": digest})) + (store / INGEST_MANIFEST_NAME).write_text(json.dumps({"corpus_sha256": digest})) # v1 store + st = corpus_status(corpus, store) + assert not st["ok"] and "chunking" in st["reason"] + (store / INGEST_MANIFEST_NAME).write_text(json.dumps({"corpus_sha256": digest, "chunking": dict(CHUNKING)})) + assert corpus_status(corpus, store)["ok"]