diff --git a/docs/bandit-recommendations.md b/docs/bandit-recommendations.md new file mode 100644 index 00000000..b16cea29 --- /dev/null +++ b/docs/bandit-recommendations.md @@ -0,0 +1,114 @@ +# Bandit route-order recommendations + +`BANDIT_MODE` defaults to `off`. An empty value also means off. An invalid value +means off and emits one warning without including the value. In this mode, +existing evaluation proposals and request dispatch keep their existing behavior. + +Set `BANDIT_MODE=shadow` to inspect a hypothetical order, or +`BANDIT_MODE=recommendation` to review an order proposal. Both modes read existing +evaluation results. They do not call providers, duplicate requests, explore paid +candidates, change task scores, or apply route configuration. No remote classifier +is used. The administrator session and existing CSRF protection are required. + +## Request a proposal + +Enable `CONFIG_SNAPSHOTS_ENABLED=true` and configure the existing private D1 +snapshot storage. Submit an administrator request with its session CSRF token: + +```http +POST /admin/workbench/shadow/propose +Content-Type: application/json + +{"route_id":"auto:chat","task_type":"chat","seed":17} +``` + +Task types are `chat`, `coding`, `extraction`, `writing`, and `reasoning`. The +optional seed defaults to zero and accepts integers from zero to 4294967295. +The response includes `mode`, `route_id`, `task_type`, `seed`, `base_revision`, +`eligible_order_before`, `eligible_order_after`, `evidence`, and `uncertainty`. +Evidence shows sample counts, decayed sample counts, quality and price coverage, +feedback counts, quality estimates, measured average costs, and 95% intervals. +The result is deterministic for the same observations, evaluation time and seed. + +A proposal can swap one adjacent pair at most. It retains exactly the configured +candidate IDs and introduces no models, tiers, credentials, or permissions. +The configured order is the review universe; request-time capability, health, +authorization and budget checks still determine which candidates may dispatch. +An unchanged order reports `insufficient_evidence` or `confidence_overlap`. + +## Evidence and uncertainty + +Each task/model cell needs at least 50 distinct stored samples and complete known +quality and cost coverage. Unknown values remain unknown, including malformed +prices and failed or truncated judgments. A known monetary zero is accepted only +as a measured value. Duplicate exports cannot increase sample counts. Invalid +tool judgments do not support a move. When three-arm evidence is present, its +existing confidence and noise-floor gate must pass, even if three-arm evaluation +is currently disabled. + +Observations decay with a seven-day half-life. Decayed quality successes and +failures update a Beta(1,1) prior; 1000 seeded posterior draws supply the 95% +quality interval. Utility is quality minus the weighted mean of +`cost_usd / (1 + cost_usd)`. Empirical cost-penalty variation widens its interval. +A move requires the promoted cell's lower utility bound to exceed the preceding +cell's upper bound. Old evidence therefore has less influence and more +uncertainty. These intervals describe the measured cohort and chosen utility; +they do not certify future quality or remove evaluation bias. + +The process holds at most 1000 task/model cells and 10000 observations, rebuilt +from the existing versioned evaluation result store. Results older than 90 days +are excluded. A purge removes their aggregate influence on the next read. Only +identifiers, timestamps, numeric judgments and costs are consumed; prompts, +answers and judge explanations are neither read for scoring nor logged. + +## Explicit feedback + +An administrator can obtain a nonce for a stored sample/model observation: + +```http +POST /admin/workbench/shadow/bandit/nonce +Content-Type: application/json + +{"sample_id":"00000000000000000000000000000001","model":"openai:example"} +``` + +Submit the returned nonce once with a numeric quality from zero to one: + +```http +POST /admin/workbench/shadow/bandit/feedback +Content-Type: application/json + +{"nonce":"","quality":1} +``` + +The nonce binds the stored observation and authenticated administrator. It +expires after 15 minutes. Feedback replaces that observation's quality without +increasing its sample count or supplying a missing price. A second review, +replayed nonce, foreign administrator, expired nonce, or removed observation is +rejected. At most 1000 outstanding nonces are kept. Nonces and explicit feedback +are process-local and disappear on restart; submit to the same running process. +No distributed or durable feedback guarantee is provided. Feedback does not +override a failed tool or noise-floor gate. + +## Review and apply + +Review the complete proposal, then create a configuration snapshot through +`POST /admin/config/snapshots` using `domain: "auto_routes"`, the proposal's +`base_revision`, and a `configuration.routes` entry containing its `route_id` and +`eligible_order_after`. Review `GET /admin/config/snapshots/{id}/diff`, then use +`POST /admin/config/snapshots/{id}/apply` with `confirm: true` and +`current_revision` equal to the reviewed base revision. This existing apply +path owns the atomic compare-and-swap check and audit record. A stale revision +returns HTTP 409. A proposal is advice; it is never an apply authorization. + +While either bandit mode is enabled, `/admin/workbench/shadow/apply` returns +HTTP 400 and directs the operator to snapshot review. A route read checks the +durable revision before and after reading fresh route bytes; a concurrent change +returns HTTP 409. Missing or invalid durable storage fails closed with HTTP 503; +disabled snapshot storage or an absent route returns HTTP 404. Invalid proposal +or feedback fields return HTTP 400. Feedback conflicts return HTTP 409, and the +nonce/feedback endpoints return HTTP 404 while bandit mode is off. + +Both direct Flask requests and administrator requests forwarded by the Worker +use these same endpoints. No additional Worker dispatch implementation, storage +migration, browser-readable response header, or provider capability is required. diff --git a/routes/shadow_eval.py b/routes/shadow_eval.py index 74bc877b..d5dcb661 100644 --- a/routes/shadow_eval.py +++ b/routes/shadow_eval.py @@ -1,6 +1,7 @@ """Administrator-only evaluation controls; retained text loads only on demand.""" from functools import partial +from types import SimpleNamespace from flask import jsonify, request @@ -12,14 +13,22 @@ from services.auto_route_service import AutoRouteService from services.intelligence_store import IntelligenceStore from services.shadow_eval_contract import validate_config -from services.shadow_eval_league import league, proposal, result_counts +from services.shadow_eval_league import bandit_proposal, league, proposal, result_counts from services.shadow_eval_runner import start_run from services.shadow_eval_sampling import sampling_counts from services.shadow_eval_store import ShadowEvalStore from services import evaluation_noise_floor as noise_floor +from services import bandit_recommendations as bandit -def current_proposal(): +def current_proposal(options=None): + selected_mode = bandit.mode() + if selected_mode != "off": + route_id, task, seed = bandit.proposal_options(options) + revision, order = bandit.read_route_revision(route_id) + document = bandit_proposal(ShadowEvalStore.results(), SimpleNamespace(id=route_id, candidates=order), + task, revision, seed=seed) + return None, {"mode": selected_mode, **document} policy = IntelligenceStore.policy() rows = league(ShadowEvalStore.results()) return policy, proposal(policy, rows, AutoRouteService.list_routes()) @@ -85,13 +94,15 @@ def shadow_purge(): @login_required def shadow_propose(): require_admin_dashboard_user() - _, document = current_proposal() + _, document = current_proposal(json_object_body() if bandit.mode() != "off" else None) return jsonify(document) @app.post("/admin/workbench/shadow/apply") @login_required def shadow_apply(): require_admin_dashboard_user() + if bandit.mode() != "off": + raise APIError("Review route-order changes through configuration snapshot apply", 400) body = json_object_body() if set(body) != {"confirm", "revision"} or body["confirm"] is not True: raise APIError("Explicitly confirm the reviewed policy proposal", 400) @@ -104,6 +115,40 @@ def shadow_apply(): raise APIError("Policy changed or is unseeded; reload and review again", 409) return jsonify({"applied": True, "auto_routes_applied": False}) + @app.post("/admin/workbench/shadow/bandit/nonce") + @login_required + def bandit_nonce(): + user = require_admin_dashboard_user() + if bandit.mode() == "off": + raise APIError("Not found", 404) + body = json_object_body() + if (set(body) != {"sample_id", "model"} or not isinstance(body["sample_id"], str) + or not bandit.ID.fullmatch(body["sample_id"]) or not isinstance(body["model"], str)): + raise APIError("Expected a stored sample_id and model", 400) + bandit.recommendations.replace_results(ShadowEvalStore.results()) + try: + nonce = bandit.recommendations.issue_nonce(bandit.observation_id(body["sample_id"], body["model"]), + bandit.config_snapshots.actor_identity(user)) + except ValueError as error: + raise APIError(str(error), 409) from None + return jsonify({"nonce": nonce, "expires_in_seconds": bandit.NONCE_TTL}) + + @app.post("/admin/workbench/shadow/bandit/feedback") + @login_required + def bandit_feedback(): + user = require_admin_dashboard_user() + if bandit.mode() == "off": + raise APIError("Not found", 404) + body = json_object_body() + if set(body) != {"nonce", "quality"} or not isinstance(body["nonce"], str) or not bandit._number(body["quality"], 0, 1): + raise APIError("Expected nonce and quality between zero and one", 400) + bandit.recommendations.replace_results(ShadowEvalStore.results()) + try: + bandit.recommendations.feedback(body["nonce"], body["quality"], bandit.config_snapshots.actor_identity(user)) + except ValueError as error: + raise APIError(str(error), 409) from None + return jsonify({"accepted": True}) + @app.post("/admin/shadow-eval/run") @csrf.exempt @api_auth_required diff --git a/services/bandit_recommendations.py b/services/bandit_recommendations.py new file mode 100644 index 00000000..90548308 --- /dev/null +++ b/services/bandit_recommendations.py @@ -0,0 +1,295 @@ +"""Bounded, content-free evidence for reviewed route-order recommendations.""" + +import hashlib +import logging +import math +import os +import random +import secrets +import threading +import time +from dataclasses import dataclass +from typing import Callable + +from error_handlers import APIError +from services import auto_route_d1, config_snapshots, evaluation_noise_floor as noise_floor +from services.auto_route_service import DEFAULT_AUTO_ROUTES +from services.shadow_eval_contract import ID, MODEL, TASKS + +MIN_SAMPLES = 50 +MAX_CELLS = 1000 +MAX_OBSERVATIONS = 10000 +HALF_LIFE = 7 * 86400 +RESULT_TTL = 90 * 86400 +NONCE_TTL = 900 +MAX_NONCES = 1000 +POSTERIOR_DRAWS = 1000 +logger = logging.getLogger(__name__) +_warned = False +_mode_lock = threading.Lock() + + +def mode(): + global _warned + value = os.environ.get("BANDIT_MODE", "").strip().lower() or "off" + if value in {"off", "shadow", "recommendation"}: + return value + with _mode_lock: + if not _warned: + logger.warning("Invalid BANDIT_MODE; route-order recommendations are off") + _warned = True + return "off" + + +def _number(value, minimum=0, maximum=None): + try: + return (type(value) in {int, float} and math.isfinite(value) and value >= minimum + and (maximum is None or value <= maximum)) + except OverflowError: + return False + + +def observation_id(sample_id, model): + return hashlib.sha256(f"{sample_id}:{model}".encode()).hexdigest()[:32] + + +@dataclass(frozen=True) +class Observation: + identifier: str + task: str + model: str + timestamp: float + quality: float | None + cost: float | None + noise_eligible: bool + + +def _cell(task, model, records, feedback, now, seed): + count = len(records) + weights = [2 ** (-(now - record.timestamp) / HALF_LIFE) for record in records] + mass = sum(weights) + qualities = [feedback.get(record.identifier, (record, record.quality))[1] for record in records] + quality_count = sum(value is not None for value in qualities) + cost_count = sum(record.cost is not None for record in records) + known = count > 0 and quality_count == cost_count == count + row = {"task_type": task, "model": model, "sample_count": count, "minimum_samples": MIN_SAMPLES, + "decayed_samples": round(mass, 6), "feedback_count": sum(record.identifier in feedback for record in records), + "coverage": {"quality": quality_count / count if count else 0, "cost": cost_count / count if count else 0}, + "quality_mean": None, "mean_cost_usd": None, "quality_ci": None, "utility_ci": None, + "noise_eligible": all(record.noise_eligible for record in records), "eligible": False} + if not known: + return row + successes = sum(weight * quality for weight, quality in zip(weights, qualities)) + costs = [record.cost for record in records] + average_cost = sum(weight * cost for weight, cost in zip(weights, costs)) / mass + if not math.isfinite(average_cost): + return row + # Cost penalties stay bounded even for very large valid monetary observations. + penalties = [cost / (1 + cost) for cost in costs] + penalty = sum(weight * cost for weight, cost in zip(weights, penalties)) / mass + variance = sum(weight * (cost - penalty) ** 2 for weight, cost in zip(weights, penalties)) / mass + error = 1.96 * math.sqrt(variance / mass) + digest = hashlib.sha256(f"{seed}:{task}:{model}".encode()).digest() + # Seeded statistical draws never generate authentication or feedback tokens. + rng = random.Random(int.from_bytes(digest, "big")) # nosec B311 + draws = sorted(rng.betavariate(1 + successes, 1 + mass - successes) for _ in range(POSTERIOR_DRAWS)) + lower, upper = draws[24], draws[974] + cost_lower, cost_upper = max(0, penalty - error), min(1, penalty + error) + row.update(quality_mean=round(successes / mass, 6), mean_cost_usd=average_cost, + quality_ci=[round(lower, 6), round(upper, 6)], + utility_ci=[round(lower - cost_upper, 6), round(upper - cost_lower, 6)], + eligible=count >= MIN_SAMPLES and row["noise_eligible"]) + return row + + +class BanditRecommendations: + """Trusted observation boundary; no provider, dispatch or policy-write methods.""" + + def __init__(self, *, clock: Callable[[], float] | None = None) -> None: + self.clock = clock or (lambda: time.time()) + self._observations: dict[str, Observation] = {} + self._feedback: dict[str, tuple[Observation, float]] = {} + self._nonces: dict[str, tuple[Observation, str, float]] = {} + self._lock = threading.RLock() + self._cell_keys: set[tuple[str, str]] = set() + + def _prune(self, now): + self._observations = {key: value for key, value in self._observations.items() + if now - RESULT_TTL <= value.timestamp <= now} + self._cell_keys = {(value.task, value.model) for value in self._observations.values()} + self._feedback = {key: value for key, value in self._feedback.items() + if self._observations.get(key) == value[0]} + self._nonces = {key: value for key, value in self._nonces.items() + if value[2] > now and self._observations.get(value[0].identifier) == value[0]} + + def observe(self, *, observation_id: str, task_type: str, model: str, observed_at: float, + quality: float | None, cost_usd: float | None, trusted: bool = False, + noise_eligible: bool = True) -> bool: + now = self.clock() + if (trusted is not True or not isinstance(observation_id, str) or not ID.fullmatch(observation_id) + or task_type not in TASKS or not config_snapshots.safe_identifier(model, MODEL) + or not _number(observed_at, now - RESULT_TTL, now) + or quality is not None and not _number(quality, 0, 1) + or cost_usd is not None and not _number(cost_usd) + or type(noise_eligible) is not bool): + return False + with self._lock: + if observation_id in self._observations or len(self._observations) >= MAX_OBSERVATIONS: + return False + if (task_type, model) not in self._cell_keys and len(self._cell_keys) >= MAX_CELLS: + return False + self._observations[observation_id] = Observation(observation_id, task_type, model, observed_at, + quality, cost_usd, noise_eligible) + self._cell_keys.add((task_type, model)) + return True + + def replace_results(self, results): + # Rebuild from the authoritative store, so purged or changed evidence cannot survive in a cache. + with self._lock: + self._observations = {} + self._cell_keys = set() + for observation in _stored_observations(results): + self.observe(**observation, trusted=True) + self._prune(self.clock()) + + def issue_nonce(self, identifier: str, actor: str) -> str: + with self._lock: + now = self.clock() + self._prune(now) + observation = self._observations.get(identifier) + if observation is None or identifier in self._feedback: + raise ValueError("Feedback observation is unavailable or already reviewed") + if len(self._nonces) >= MAX_NONCES: + raise ValueError("Feedback nonce limit reached") + nonce = secrets.token_urlsafe(32) + self._nonces[nonce] = (observation, actor, now + NONCE_TTL) + return nonce + + def feedback(self, nonce: str, quality: float, actor: str) -> None: + if not isinstance(nonce, str) or not _number(quality, 0, 1): + raise ValueError("Invalid feedback") + with self._lock: + self._prune(self.clock()) + entry = self._nonces.get(nonce) + if entry is None or entry[1] != actor or entry[0].identifier in self._feedback: + raise ValueError("Feedback nonce expired, used or unavailable") + observation = entry[0] + self._feedback[observation.identifier] = (observation, quality) + del self._nonces[nonce] + + def cells(self, *, seed=0): + with self._lock: + now = self.clock() + self._prune(now) + groups: dict[tuple[str, str], list[Observation]] = {} + for observation in self._observations.values(): + groups.setdefault((observation.task, observation.model), []).append(observation) + return [_cell(task, model, sorted(records, key=lambda value: value.identifier), self._feedback, now, seed) + for (task, model), records in sorted(groups.items())] + + def propose(self, *, task_type, eligible_order, base_revision, seed=0): + if (task_type not in TASKS or not config_snapshots.revision(base_revision) + or type(seed) is not int or not 0 <= seed < 2**32 + or not isinstance(eligible_order, (list, tuple)) or not 1 <= len(eligible_order) <= 16 + or any(not config_snapshots.safe_identifier(model, MODEL) for model in eligible_order) + or len(set(eligible_order)) != len(eligible_order)): + raise ValueError("Invalid bandit proposal options") + by_model = {row["model"]: row for row in self.cells(seed=seed) if row["task_type"] == task_type} + evidence = [by_model.get(model) or _cell(task_type, model, [], {}, self.clock(), seed) for model in eligible_order] + after = list(eligible_order) + moves = [] + for index in range(1, len(evidence)): + before, candidate = evidence[index - 1:index + 1] + if before["eligible"] and candidate["eligible"]: + improvement = candidate["utility_ci"][0] - before["utility_ci"][1] + if improvement > 0: + moves.append((improvement, -index, index)) + if moves: + index = max(moves)[2] + after[index - 1], after[index] = after[index], after[index - 1] + reason = "supported_move" if moves else "confidence_overlap" if all(row["eligible"] for row in evidence) else "insufficient_evidence" + return {"base_revision": base_revision, "task_type": task_type, "seed": seed, + "eligible_order_before": list(eligible_order), "eligible_order_after": after, "evidence": evidence, + "uncertainty": {"confidence": 0.95, "half_life_seconds": HALF_LIFE, "reason": reason, + "utility": "quality - mean(cost_usd / (1 + cost_usd))", "posterior_draws": POSTERIOR_DRAWS}} + + +def _noise_gate(results): + affected = {(result.get("task_type"), result.get(f"{side}_model")) for result in results if "noise_floor" in result + for side in ("candidate", "production")} + try: + records = [result for result in results if isinstance(result.get("noise_floor"), dict) + and type(result["noise_floor"].get("seed")) is int + and 0 <= result["noise_floor"]["seed"] < 2**32 + and type(result["noise_floor"].get("bootstrap_draws")) is int + and noise_floor.DEFAULT_DRAWS <= result["noise_floor"]["bootstrap_draws"] <= noise_floor.MAX_DRAWS] + # Sparse cohorts cannot pass the existing gate. Avoid spending draws on each isolated seed. + groups = noise_floor._cohorts(records) + eligible_samples = {sample for group in groups.values() if len(group) >= noise_floor.MIN_SAMPLES for sample in group} + summaries = noise_floor.summarize([record for record in records if record["sample_id"] in eligible_samples]) + except (KeyError, TypeError, ValueError, ZeroDivisionError): + return affected, set() + allowed = set() + for key in affected: + cohorts = [row for row in summaries if row["task_type"] == key[0] + and key[1] in {row["candidate_model"], row["baseline_model"]}] + if cohorts and all(row["eligible"] for row in cohorts): + allowed.add(key) + return affected, allowed + + +def _stored_observations(results): + # Store metadata establishes provenance; public request bodies never reach this adapter. + records = [result for result in list(results)[:MAX_OBSERVATIONS] if isinstance(result, dict) + and isinstance(result.get("_id"), str) and ID.fullmatch(result["_id"]) + and isinstance(result.get("sample_id"), str) and ID.fullmatch(result["sample_id"]) + and _number(result.get("_created_at")) and result.get("task_type") in TASKS + and all(config_snapshots.safe_identifier(result.get(f"{side}_model"), MODEL) + for side in ("candidate", "production"))] + records.sort(key=lambda value: (-value["_created_at"], value["_id"])) + affected, allowed = _noise_gate(records) + for result in records: + if result.get("candidate_model") == result.get("production_model"): + continue + quality = {"win": 1.0, "loss": 0.0, "tie": 0.5}.get(result.get("outcome")) + validity = result.get("tool_validity") + valid = quality is not None and not result.get("candidate_truncated") and (validity is None or isinstance(validity, dict) and all( + isinstance(value, dict) and value.get("invalid") == 0 for value in validity.values())) + costs = result.get("costs") + for side in ("candidate", "production"): + model = result.get(f"{side}_model") + if not isinstance(model, str): + continue + key = (result["task_type"], model) + cost = costs.get(side) if isinstance(costs, dict) else None + yield {"observation_id": observation_id(result["sample_id"], model), "task_type": key[0], "model": model, + "observed_at": result["_created_at"], "quality": (quality if side == "candidate" else 1 - quality) + if quality is not None and valid else None, + "cost_usd": cost if _number(cost) else None, "noise_eligible": valid and (key not in affected or key in allowed)} + + +recommendations = BanditRecommendations() + + +def read_route_revision(route_id): + """Read fresh route bytes between revision checks; never use the routing fallback cache.""" + revision = config_snapshots.call("snapshot_list")["current_revision"] + try: + routes = auto_route_d1._routes(auto_route_d1.snapshot_request("list")) + except ValueError: + raise APIError("Bandit route evidence is unavailable", 503, {"error": "bandit_storage_unavailable"}) from None + if config_snapshots.call("snapshot_list")["current_revision"] != revision: + raise APIError("Configuration changed; reload and review again", 409, {"error": "revision_conflict"}) + order = routes.get(route_id, (DEFAULT_AUTO_ROUTES.get(route_id), None))[0] + if order is None: + raise APIError("Auto route not found", 404) + return revision, order + + +def proposal_options(body): + if (not isinstance(body, dict) or set(body) - {"route_id", "task_type", "seed"} + or not isinstance(body.get("route_id"), str) or not auto_route_d1._ROUTE_ID.fullmatch(body["route_id"]) + or body.get("task_type") not in TASKS or type(body.get("seed", 0)) is not int + or not 0 <= body.get("seed", 0) < 2**32): + raise APIError("Expected route_id, task_type and optional integer seed", 400) + return body["route_id"], body["task_type"], body.get("seed", 0) diff --git a/services/shadow_eval_league.py b/services/shadow_eval_league.py index 250926b0..bad6cffa 100644 --- a/services/shadow_eval_league.py +++ b/services/shadow_eval_league.py @@ -6,6 +6,7 @@ from services.shadow_eval_contract import encoded from services import evaluation_noise_floor as noise_floor +from services import bandit_recommendations as bandit # Anchor Elo differences to existing absolute policy scores (100 Elo = 10 points). @@ -114,3 +115,11 @@ def result_counts(results): else: counts["failed"] += 1 return counts + + +def bandit_proposal(results, route, task_type, base_revision, *, seed=0, recommender=None): + recommender = recommender or bandit.recommendations + recommender.replace_results(results) + document = recommender.propose(task_type=task_type, eligible_order=list(route.candidates), + base_revision=base_revision, seed=seed) + return {"route_id": route.id, **document} diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 2c079f26..5a4479d7 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -149,6 +149,8 @@ "app.get('/admin/workbench/shadow/samples/')", "app.post('/admin/shadow-eval/run')", "app.post('/admin/workbench/shadow/apply')", + "app.post('/admin/workbench/shadow/bandit/feedback')", + "app.post('/admin/workbench/shadow/bandit/nonce')", "app.post('/admin/workbench/shadow/propose')", "app.post('/admin/workbench/shadow/purge')", "app.route('/admin/workbench/shadow/config', methods=['GET', 'POST'])" diff --git a/tests/test_bandit_recommendations.py b/tests/test_bandit_recommendations.py new file mode 100644 index 00000000..01640d8d --- /dev/null +++ b/tests/test_bandit_recommendations.py @@ -0,0 +1,296 @@ +"""Read-only ordering, bounded evidence and administrator feedback contracts.""" + +import copy +import importlib +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from flask import Flask +from flask_wtf.csrf import CSRFError, CSRFProtect + +NOW = 2_000_000_000.0 +BEFORE = ["openai:production", "openai:candidate"] + + +@pytest.fixture(autouse=True) +def isolated(monkeypatch): + monkeypatch.setenv("BANDIT_MODE", "") + monkeypatch.setenv("CONFIG_SNAPSHOTS_ENABLED", "true") + monkeypatch.setenv("SHADOW_EVAL_NOISE_FLOOR_ENABLED", "") + + +def engine(): + module = importlib.import_module("services.bandit_recommendations") + return module.BanditRecommendations(clock=lambda: NOW) + + +def populate(value, count=50, *, quality=(0.0, 1.0), costs=(0.02, 0.01), age=0): + for side, model in enumerate(BEFORE): + for index in range(count): + assert value.observe(observation_id=f"{side * 10000 + index:032x}", task_type="chat", model=model, + observed_at=NOW - age, quality=quality[side], cost_usd=costs[side], trusted=True) + + +def propose(value, **changes): + return value.propose(task_type="chat", eligible_order=BEFORE, base_revision=7, seed=17, **changes) + + +@pytest.mark.parametrize("count", [0, 1, 49]) +def test_sparse_cells_cannot_move(count): + value = engine() + populate(value, count) + document = propose(value) + assert document["eligible_order_after"] == BEFORE + assert document["uncertainty"]["reason"] == "insufficient_evidence" + + +@pytest.mark.parametrize("field,unknown", [("costs", (0.02, None)), ("quality", (0.0, None))]) +def test_unknown_is_not_favourable_zero(field, unknown): + value = engine() + populate(value, **{field: unknown}) + document = propose(value) + assert document["eligible_order_after"] == BEFORE + row = document["evidence"][1] + assert row["utility_ci"] is None + assert row["coverage"]["cost" if field == "costs" else "quality"] == 0 + + +def test_seeded_confidence_clear_effect_and_null(): + value = engine() + populate(value) + document = propose(value) + assert document == propose(value) + assert document["eligible_order_after"] == BEFORE[::-1] + assert document["base_revision"] == 7 + assert document["uncertainty"]["confidence"] == 0.95 + assert document["evidence"][1]["utility_ci"][0] > document["evidence"][0]["utility_ci"][1] + null = engine() + populate(null, quality=(0.5, 0.5), costs=(0.01, 0.01)) + assert propose(null)["eligible_order_after"] == BEFORE + + +def test_half_life_and_single_adjacent_step(): + value = engine() + populate(value, count=100, age=7 * 86400) + assert propose(value)["evidence"][0]["decayed_samples"] == 50 + models = [*BEFORE, "openai:third"] + for index in range(100): + value.observe(observation_id=f"{30000 + index:032x}", task_type="chat", model=models[2], + observed_at=NOW, quality=1, cost_usd=0, trusted=True) + document = value.propose(task_type="chat", eligible_order=models, base_revision=7, seed=17) + moved = [i for i, model in enumerate(models) if document["eligible_order_after"][i] != model] + assert len(moved) == 2 and moved[1] - moved[0] == 1 + assert set(document["eligible_order_after"]) == set(models) + + +def test_untrusted_duplicate_invalid_and_bounded_cells(): + value = engine() + record = dict(observation_id="a" * 32, task_type="chat", model=BEFORE[0], observed_at=NOW, quality=1, cost_usd=0) + assert not value.observe(**record) + assert value.observe(**record, trusted=True) + assert not value.observe(**record, trusted=True) + for changes in ({"quality": float("nan")}, {"cost_usd": -1}, {"quality": True}, {"observed_at": NOW + 1}): + assert not value.observe(**{**record, "observation_id": "b" * 32, **changes}, trusted=True) + for index in range(1000): + accepted = value.observe(**{**record, "observation_id": f"{index:032x}", "model": f"openai:m{index}"}, trusted=True) + assert accepted is (index < 999) + assert len(value.cells()) == 1000 + + +def test_feedback_is_single_use_actor_bound_and_does_not_inflate_samples(): + value = engine() + populate(value, quality=(None, None)) + identifier = f"{10000:032x}" + nonce = value.issue_nonce(identifier, "operator") + with pytest.raises(ValueError): + value.feedback(nonce, 1, "another-operator") + value.feedback(nonce, 1, "operator") + with pytest.raises(ValueError): + value.feedback(nonce, 1, "operator") + cell = propose(value)["evidence"][1] + assert cell["sample_count"] == 50 and cell["feedback_count"] == 1 + assert cell["coverage"]["quality"] == 0.02 + + +def records(count=50, **changes): + return [{"_id": f"{index + 1:032x}", "_created_at": NOW, "sample_id": f"{index:032x}", + "task_type": "chat", "candidate_model": BEFORE[1], "production_model": BEFORE[0], + "outcome": "win", "costs": {"candidate": 0.01, "production": 0.02}, + "latencies": {"candidate": 100, "production": 100}, + "tool_validity": None, **changes} for index in range(count)] + + +def test_stored_result_adapter_requires_metadata_deduplicates_and_respects_noise_gate(monkeypatch): + module = importlib.import_module("services.shadow_eval_league") + route = SimpleNamespace(id="auto:review", candidates=BEFORE) + value = engine() + data = records() + document = module.bandit_proposal(data + data, route, "chat", 7, seed=17, recommender=value) + assert document["eligible_order_after"] == BEFORE[::-1] + assert all(row["sample_count"] == 50 for row in document["evidence"]) + assert module.bandit_proposal(records(_created_at=None), route, "chat", 7, + recommender=value)["eligible_order_after"] == BEFORE + blocked = records(noise_floor={"version": 1}) + monkeypatch.setattr(module.noise_floor, "summarize", lambda data: [{"task_type": "chat", + "candidate_model": BEFORE[1], "baseline_model": BEFORE[0], "eligible": False}]) + assert module.bandit_proposal(blocked, route, "chat", 7, recommender=value)["eligible_order_after"] == BEFORE + + +@pytest.fixture +def browser(monkeypatch): + route = importlib.import_module("routes.shadow_eval") + module = route.bandit + monkeypatch.setattr(module, "recommendations", engine()) + monkeypatch.setattr(module.time, "time", lambda: NOW) + app = Flask(__name__) + app.config.update(SECRET_KEY="synthetic-bandit-session", TESTING=True, WTF_CSRF_ENABLED=False, + API_BASE_URLS={"openai": "https://synthetic.invalid"}) + importlib.import_module("error_handlers").init_error_handlers(app) + csrf = CSRFProtect(app) + app.register_error_handler(CSRFError, importlib.import_module("routes.csrf_errors").handle_csrf_error) + route.register_shadow_eval_routes(app, csrf, Mock(), Mock(), Mock()) + snapshots = importlib.import_module("routes.config_snapshots") + snapshots.register_config_snapshot_routes(app) + monkeypatch.setattr(route.login_required.__globals__["AuthService"], "is_authenticated", lambda: True) + monkeypatch.setattr(route.require_admin_dashboard_user.__globals__["AuthService"], "get_current_user", + lambda: {"username": "operator", "is_admin": True}) + monkeypatch.setattr(snapshots.require_admin_dashboard_user.__globals__["AuthService"], "get_current_user", + lambda: {"username": "operator", "is_admin": True}) + monkeypatch.setattr(route.ShadowEvalStore, "results", lambda: records()) + monkeypatch.setattr(route.AutoRouteService, "list_routes", lambda: [SimpleNamespace(id="auto:review", candidates=BEFORE)]) + no_write = Mock(side_effect=AssertionError("No route policy writes")) + no_dispatch = Mock(side_effect=AssertionError("No provider requests")) + monkeypatch.setattr(route.ShadowEvalStore, "apply", no_write) + monkeypatch.setattr(route, "start_run", no_dispatch) + monkeypatch.setattr(route, "dispatch_unified_chat_completion", no_dispatch) + monkeypatch.setattr(route.bandit.config_snapshots, "call", lambda operation: {"current_revision": 7}) + monkeypatch.setattr(route.bandit.auto_route_d1, "snapshot_request", lambda operation: {"version": 1, "routes": [ + {"route_id": "auto:review", "candidates": BEFORE, "updated_at": "2026-10-09T00:00:00Z"}]}) + return app.test_client(), route, no_write, no_dispatch, snapshots + + +@pytest.mark.parametrize("mode", ["shadow", "recommendation"]) +def test_registered_proposal_no_write_no_extra_calls_and_reviewed_cas(browser, monkeypatch, mode): + client, route, no_write, no_dispatch, snapshots = browser + monkeypatch.setenv("BANDIT_MODE", mode) + response = client.post("/admin/workbench/shadow/propose", json={"route_id": "auto:review", "task_type": "chat", "seed": 17}) + assert response.status_code == 200 + document = response.json + assert document["mode"] == mode and document["eligible_order_after"] == BEFORE[::-1] + assert document["base_revision"] == 7 + assert client.post("/admin/workbench/shadow/apply", json={"confirm": True, "revision": "anything"}).status_code == 400 + no_write.assert_not_called() + no_dispatch.assert_not_called() + calls = [] + def stale_apply(operation, **values): + calls.append((operation, values)) + raise importlib.import_module("error_handlers").APIError("Snapshot request was rejected", 409, {"error": "revision_conflict"}) + monkeypatch.setattr(snapshots.config_snapshots, "call", stale_apply) + assert client.post("/admin/config/snapshots/" + "a" * 32 + "/apply", + json={"confirm": True, "current_revision": document["base_revision"]}).status_code == 409 + assert calls[0][0] == "snapshot_apply" and calls[0][1]["current_revision"] == 7 + + +def test_off_default_empty_and_bad_mode_preserve_proposal_bytes(browser, monkeypatch, caplog): + client, route, _, _, _ = browser + from tests.test_intelligence_policy import candidate, policy + base = policy(candidates=[candidate(model) for model in BEFORE]) + monkeypatch.setattr(route.IntelligenceStore, "policy", lambda: copy.deepcopy(base)) + monkeypatch.setattr(route.bandit, "_warned", False) + monkeypatch.setattr(route.bandit.config_snapshots, "call", Mock(side_effect=AssertionError("No new storage"))) + expected = client.post("/admin/workbench/shadow/propose", json={}).data + assert b'"policy_diff"' in expected + for mode in ("off", "", "malformed-private-value", "malformed-private-value"): + monkeypatch.setenv("BANDIT_MODE", mode) + assert client.post("/admin/workbench/shadow/propose", json={}).data == expected + assert sum("BANDIT_MODE" in record.message for record in caplog.records) == 1 + assert "malformed-private-value" not in caplog.text + route.bandit.config_snapshots.call.assert_not_called() + + +def test_registered_feedback_and_csrf_admin_replay_guards(browser, monkeypatch): + client, route, no_write, no_dispatch, _ = browser + assert client.post("/admin/workbench/shadow/bandit/nonce", json={}).status_code == 404 + monkeypatch.setenv("BANDIT_MODE", "recommendation") + response = client.post("/admin/workbench/shadow/bandit/nonce", json={"sample_id": "0" * 32, "model": BEFORE[1]}) + assert response.status_code == 200 + body = {"nonce": response.json["nonce"], "quality": 0} + assert client.post("/admin/workbench/shadow/bandit/feedback", json=body).status_code == 200 + assert client.post("/admin/workbench/shadow/bandit/feedback", json=body).status_code == 409 + client.application.config["WTF_CSRF_ENABLED"] = True + assert client.post("/admin/workbench/shadow/bandit/feedback", json=body).status_code == 400 + client.application.config["WTF_CSRF_ENABLED"] = False + monkeypatch.setattr(route.require_admin_dashboard_user.__globals__["AuthService"], "get_current_user", + lambda: {"username": "operator", "is_admin": False}) + assert client.post("/admin/workbench/shadow/propose", json={}).status_code == 403 + no_write.assert_not_called() + no_dispatch.assert_not_called() + + +def test_revision_race_and_missing_storage_fail_closed(browser, monkeypatch): + client, route, _, _, _ = browser + monkeypatch.setenv("BANDIT_MODE", "recommendation") + monkeypatch.setattr(route.bandit.config_snapshots, "call", Mock(side_effect=[{"current_revision": 7}, {"current_revision": 8}])) + body = {"route_id": "auto:review", "task_type": "chat"} + assert client.post("/admin/workbench/shadow/propose", json=body).status_code == 409 + monkeypatch.setattr(route.bandit.auto_route_d1, "snapshot_request", lambda operation: {"version": 1}) + monkeypatch.setattr(route.bandit.config_snapshots, "call", lambda operation: {"current_revision": 7}) + assert client.post("/admin/workbench/shadow/propose", json=body).status_code == 503 + + +def test_nonce_expiry_concurrency_and_purged_evidence(): + from concurrent.futures import ThreadPoolExecutor + value = engine() + populate(value) + nonce = value.issue_nonce(f"{10000:032x}", "operator") + def submit(): + try: + value.feedback(nonce, 1, "operator") + return True + except ValueError: + return False + with ThreadPoolExecutor(max_workers=2) as pool: + assert sum(pool.map(lambda unused: submit(), range(2))) == 1 + expired = value.issue_nonce(f"{10001:032x}", "operator") + value.clock = lambda: NOW + 901 + with pytest.raises(ValueError): + value.feedback(expired, 1, "operator") + value.clock = lambda: NOW + removed = value.issue_nonce(f"{10002:032x}", "operator") + value.replace_results([]) + with pytest.raises(ValueError): + value.feedback(removed, 1, "operator") + assert value.cells() == [] + + +@pytest.mark.parametrize("body", [{}, {"route_id": "auto:review", "task_type": "chat", "seed": True}, + {"route_id": "auto:review", "task_type": "chat", "trusted": True}]) +def test_invalid_or_untrusted_proposal_input_rejected_before_storage(browser, monkeypatch, body): + client, route, _, _, _ = browser + monkeypatch.setenv("BANDIT_MODE", "recommendation") + storage = Mock(side_effect=AssertionError("No storage on invalid input")) + monkeypatch.setattr(route.bandit.config_snapshots, "call", storage) + assert client.post("/admin/workbench/shadow/propose", json=body).status_code == 400 + storage.assert_not_called() + + +@pytest.mark.parametrize("noise,move", [("tie", True), ("win", False)]) +def test_real_three_arm_noise_gate_even_when_evaluation_is_off(noise, move): + from tests.test_evaluation_noise_floor import three_result + module = importlib.import_module("services.shadow_eval_league") + data = [three_result(index, ab=noise, _id=f"{index + 1:032x}", _created_at=NOW) for index in range(50)] + route = SimpleNamespace(id="auto:review", candidates=BEFORE) + document = module.bandit_proposal(data, route, "chat", 7, recommender=engine()) + assert (document["eligible_order_after"] != BEFORE) is move + + +def test_content_is_excluded_and_malformed_cost_stays_unknown(caplog): + module = importlib.import_module("services.shadow_eval_league") + data = records(prompt="private-content-marker", answer="private-content-marker") + data[0]["costs"]["candidate"] = float("inf") + route = SimpleNamespace(id="auto:review", candidates=BEFORE) + document = module.bandit_proposal(data, route, "chat", 7, recommender=engine()) + assert document["eligible_order_after"] == BEFORE + assert document["evidence"][1]["coverage"]["cost"] == 0.98 + assert "private-content-marker" not in str(document) + caplog.text diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..9574bdd2 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,7 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "BANDIT_MODE", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS",