diff --git a/docs/prompt-cache-cost.md b/docs/prompt-cache-cost.md new file mode 100644 index 00000000..c479ea81 --- /dev/null +++ b/docs/prompt-cache-cost.md @@ -0,0 +1,53 @@ +# Prompt cache usage and price buckets + +`PROMPT_CACHE_USAGE_BUCKETS_ENABLED` defaults to false. Empty values use the default. +Malformed flags or metadata disable buckets and produce one warning per setting without its value. +Disabled deployments use the existing rows, SQL, costs and protocol usage translations; migration 0016 is unnecessary. + +Enable with `PROMPT_CACHE_USAGE_BUCKETS_ENABLED=true` after applying `0016_usage_buckets.sql`. +`MODEL_PRICING_USD_PER_MILLION` keeps input/output/request prices and accepts explicit +`cache_read` and `cache_write` prices per million tokens. For example: + +```json +{"openai:example":{"input":2,"output":8,"cache_read":0.5,"cache_write":3}} +``` + +`PROMPT_CACHE_PRICE_METADATA_JSON` defaults to `{}`. It may supply missing bucket prices +for an explicitly priced model, using the same model keys and price names. Explicit +prices win, including zero and invalid prices. Metadata alone cannot price a model. +No provider-wide discount factors are assumed. + +OpenAI prompt/input counts include cache tokens. Anthropic input counts exclude reads +and writes. Only these explicit usage conventions are normalized; unknown cache reads +remain null. An OpenAI adapter has no cache-write category unless `cache_write_tokens` +is explicitly reported. Ordinary input is null when the inclusive total cannot safely +be split. Negative, boolean, string and counts above 2^53-1 remain unknown. +SSE observations merge valid components across events with a 64 KiB line bound. + +Rows add nullable ordinary-input/cache-read/cache-write counts alongside the existing +output count, four costs in microUSD, `bucket_basis` (measured/estimated/unknown) and +`bucket_source` (openai/anthropic/request_estimate/unknown). No missing count becomes +zero. A measured zero costs zero without a price. Unknown positive-count prices leave +the total null while known component costs remain visible. Decimal arithmetic is bounded +at 10^12 microUSD per component and total (one million USD). Costs may contain fractional +microUSD; totals are computed before conversion to floats. Estimates are not invoices. +Gateway response-cache hits have no provider charge. + +SQLite adds nullable columns locally only when enabled and keeps old rows readable. +The control-plane PostgreSQL adapter also adds nullable columns when enabled; it has +not been validated against a live PostgreSQL database. D1 needs migration 0016 before +enabling the feature. The migration adds nullable columns to the existing `usage_events` +table and never replaces it. + +In D1 mode the Worker's private ledger writes the bucket columns in the same batch +transaction as the base row, with the same batch guard and daily rollups. If the +extended write fails, the base row is still saved and the call returns HTTP 503 JSON +`usage_buckets_unavailable` with `cache-control: no-store`. A retry with the same batch +ID does not duplicate the base row. This is a private storage error, not a reason to +replay generation. Set `PROMPT_CACHE_USAGE_BUCKETS_ENABLED` on both the Worker and the +Container. Native Worker generations record base usage only, without cache buckets. + +Buckets follow the existing usage-event retention and contain no prompt, response or key +content. Daily rollups retain their existing total-cost schema; per-bucket history is in +raw events. No billing, retention policy, live-provider metering or migration deployment +is performed here. diff --git a/intelligence-migrations/0016_usage_buckets.sql b/intelligence-migrations/0016_usage_buckets.sql new file mode 100644 index 00000000..703ce113 --- /dev/null +++ b/intelligence-migrations/0016_usage_buckets.sql @@ -0,0 +1,10 @@ +-- Nullable prompt cache buckets on the 0007 usage ledger; output_tokens already exists. +ALTER TABLE usage_events ADD COLUMN ordinary_input_tokens INTEGER; +ALTER TABLE usage_events ADD COLUMN cache_read_input_tokens INTEGER; +ALTER TABLE usage_events ADD COLUMN cache_write_input_tokens INTEGER; +ALTER TABLE usage_events ADD COLUMN ordinary_input_cost_microusd REAL; +ALTER TABLE usage_events ADD COLUMN cache_read_input_cost_microusd REAL; +ALTER TABLE usage_events ADD COLUMN cache_write_input_cost_microusd REAL; +ALTER TABLE usage_events ADD COLUMN output_cost_microusd REAL; +ALTER TABLE usage_events ADD COLUMN bucket_basis TEXT; +ALTER TABLE usage_events ADD COLUMN bucket_source TEXT; diff --git a/package.json b/package.json index 42ee5f27..4b42830a 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_prompt_cache_cost_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/services/cost_service.py b/services/cost_service.py index 614586d2..7ff67e27 100644 --- a/services/cost_service.py +++ b/services/cost_service.py @@ -131,3 +131,10 @@ def estimate( + Decimal(safe_output_tokens) * prices["output"] ) / Decimal(1_000_000) + Decimal(cls._token_count(requests)) * prices.get("request", Decimal(0)) return float(total.quantize(Decimal("0.0000000001"))) + + @classmethod + def price_buckets(cls, model_id, usage, *, requests: int = 1) -> dict: + """Return nullable bucket costs without changing legacy estimate semantics.""" + from services.prompt_cache_cost import price_buckets + + return price_buckets(model_id, usage, requests=requests) diff --git a/services/prompt_cache_cost.py b/services/prompt_cache_cost.py new file mode 100644 index 00000000..ab78ea62 --- /dev/null +++ b/services/prompt_cache_cost.py @@ -0,0 +1,232 @@ +"""Opt-in cache bucket normalization and bounded catalog price estimates.""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation, localcontext +from functools import lru_cache +from typing import Any + +from services.usage_types import StreamUsageObserver, UsageObservation + +logger = logging.getLogger(__name__) +MAX_TOKENS = 2**53 - 1 +MAX_MICROUSD = 1_000_000_000_000 +TOKEN_FIELDS = ("ordinary_input_tokens", "cache_read_input_tokens", "cache_write_input_tokens", "output_tokens") +COST_FIELDS = ("ordinary_input_cost_microusd", "cache_read_input_cost_microusd", + "cache_write_input_cost_microusd", "output_cost_microusd") +ROW_FIELDS = TOKEN_FIELDS[:3] + COST_FIELDS + ("bucket_basis", "bucket_source") +_warned: set[str] = set() + + +def _warn(name: str) -> None: + if name not in _warned: + _warned.add(name) + logger.warning("Invalid %s; prompt cache buckets disabled", name) + + +@lru_cache(maxsize=8) +def _metadata(raw: str) -> dict | None: + if len(raw) > 262144: + return None + try: + value = json.loads(raw or "{}") + except (ValueError, RecursionError): + return None + if not isinstance(value, dict) or any(not isinstance(v, dict) for v in value.values()): + return None + if any(_price(price) is None for entry in value.values() for price in entry.values()): + return None + return value + + +def enabled() -> bool: + flag = os.environ.get("PROMPT_CACHE_USAGE_BUCKETS_ENABLED", "").strip().lower() + if flag in {"", "false", "0", "no", "off"}: + return False + if flag not in {"true", "1", "yes", "on"}: + _warn("PROMPT_CACHE_USAGE_BUCKETS_ENABLED") + return False + if _metadata(os.environ.get("PROMPT_CACHE_PRICE_METADATA_JSON", "").strip()) is None: + _warn("PROMPT_CACHE_PRICE_METADATA_JSON") + return False + return True + + +def count(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= MAX_TOKENS else None + + +def _sum(values: tuple) -> int | None: + return count(sum(values)) if all(value is not None for value in values) else None + + +@dataclass(frozen=True) +class CacheObservation(UsageObservation): + raw_input: int | None = None + cache_read: int | None = None + cache_write: int | None = None + source: str = "unknown" + + @classmethod + def from_body(cls, value: Any) -> CacheObservation | None: + if not isinstance(value, dict): + return None + native = value + if not isinstance(native.get("usage"), dict): + native = next((value[key] for key in ("response", "message") + if isinstance(value.get(key), dict) and isinstance(value[key].get("usage"), dict)), {}) + usage = native.get("usage") + if not isinstance(usage, dict): + return None + typed = UsageObservation.from_body(value) + if typed is None: + return None + anthropic = native.get("type") == "message" or "message" in value or str(value.get("type", "")).startswith("message_") or any( + key in usage for key in ("cache_read_input_tokens", "cache_creation_input_tokens")) + details = usage.get("prompt_tokens_details", usage.get("input_tokens_details")) + details = details if isinstance(details, dict) else {} + raw = count(typed.input_tokens) + read = count(usage.get("cache_read_input_tokens")) if anthropic else count(details.get("cached_tokens")) + write = count(usage.get("cache_creation_input_tokens")) if anthropic else count(details.get("cache_write_tokens", 0 if raw is not None or details else None)) + source = "anthropic" if anthropic else "openai" if raw is not None or details else "unknown" + total = _sum((raw, read, write)) if anthropic else raw + return cls(total, count(typed.output_tokens), typed.basis, typed.provenance, raw, read, write, source) + + def merge(self, later: UsageObservation) -> CacheObservation: + if not isinstance(later, CacheObservation): + return self + def latest(old, new): + return new if new is not None else old + raw = latest(self.raw_input, later.raw_input) + read = latest(self.cache_read, later.cache_read) + # An output-only delta must not imply a new cache-write observation. + write = latest(self.cache_write, later.cache_write) if later.source != "unknown" else self.cache_write + source = later.source if later.source != "unknown" else self.source + total = _sum((raw, read, write)) if source == "anthropic" else raw + output = latest(self.output_tokens, later.output_tokens) + known = any(v is not None for v in (raw, read, write, output)) + return CacheObservation(total, output, "provider" if known else "unknown", ("provider",) if known else (), + raw, read, write, source) + + +def buckets_from(usage: UsageObservation | None) -> dict: + ordinary, read, write, output, source = None, None, None, None, "unknown" + if isinstance(usage, CacheObservation): + read, write, output, source = usage.cache_read, usage.cache_write, usage.output_tokens, usage.source + if source == "anthropic": + ordinary = usage.raw_input + elif usage.raw_input is not None and read is not None and write is not None: + ordinary = count(usage.raw_input - read - write) + elif usage is not None: + output = count(usage.output_tokens) + basis = "measured" if any(v is not None for v in (ordinary, read, write, output)) else "unknown" + if usage is not None and usage.basis == "estimated": + basis, source = "estimated", "request_estimate" + return dict(zip(TOKEN_FIELDS, (ordinary, read, write, output)), bucket_basis=basis, bucket_source=source) + + +class CacheStreamObserver(StreamUsageObserver): + _pending: bytes + _oversized: bool + observation: UsageObservation | None + + def _line(self) -> None: + line, self._pending = self._pending.strip(), b"" + if self._oversized: + self._oversized = False + return + if not line.startswith(b"data:") or b'"usage"' not in line: + return + try: + found = CacheObservation.from_body(json.loads(line[5:])) + except (ValueError, RecursionError): + return + if found is not None: + self.observation = self.observation.merge(found) if self.observation is not None else found + + +def _entry(table: dict, model: str) -> dict | None: + provider = model.split(":", 1)[0] if ":" in model else "" + return next((table[key] for key in (model, f"{provider}:*" if provider else "", "*") + if key and isinstance(table.get(key), dict)), None) + + +def _price(value: Any) -> Decimal | None: + try: + number = Decimal(str(value)) + except (InvalidOperation, ValueError, TypeError): + return None + return number if number.is_finite() and 0 <= number <= MAX_MICROUSD else None + + +def price_buckets(model: str | None, usage: UsageObservation | None, *, requests: int = 1) -> dict: + buckets = buckets_from(usage) + result = {**buckets, **dict.fromkeys(COST_FIELDS), "cost_usd": None} + try: + table = json.loads(os.environ.get("MODEL_PRICING_USD_PER_MILLION", "") or "{}") + except (ValueError, RecursionError): + return result + if not isinstance(table, dict) or not model: + return result + model = model.strip().lower() + table = {key.strip().lower(): value for key, value in table.items() if isinstance(key, str)} + explicit = _entry(table, model) + if explicit is None: + return result + metadata = _metadata(os.environ.get("PROMPT_CACHE_PRICE_METADATA_JSON", "").strip()) or {} + supplement = metadata.get(model, {}) + names = ("input", "cache_read", "cache_write", "output") + aliases = {"input": "input_cost_per_million", "output": "output_cost_per_million"} + flat_only = "request" in explicit and not any(key in explicit for key in (*names, *aliases.values())) + costs = [] + with localcontext() as ctx: + ctx.prec = 40 + for tokens, cost_name, name in zip(TOKEN_FIELDS, COST_FIELDS, names): + alias = aliases.get(name, name) + value = explicit.get(name, explicit.get(alias, supplement.get(name))) + unit = Decimal(0) if flat_only else _price(value) + amount = buckets[tokens] + if amount == 0 or unit == 0: + cost = Decimal(0) + elif amount is None or unit is None: + cost = None + else: + cost = Decimal(amount) * unit + if cost > MAX_MICROUSD: + cost = None + result[cost_name] = float(cost) if cost is not None else None + costs.append(cost) + flat = _price(explicit.get("request", 0)) + if flat is not None and count(requests) is not None and all(v is not None for v in costs): + total = sum((cost for cost in costs if cost is not None), Decimal(0)) + flat * requests * 1_000_000 + if total <= MAX_MICROUSD: + result["cost_usd"] = float(total / 1_000_000) + return result + + +def anthropic_to_chat(usage: dict) -> dict: + observed = CacheObservation.from_body({"type": "message", "usage": usage}) or CacheObservation() + return {"prompt_tokens": observed.input_tokens, "completion_tokens": observed.output_tokens, + "total_tokens": _sum((observed.input_tokens, observed.output_tokens)), + "prompt_tokens_details": {"cached_tokens": observed.cache_read, "cache_write_tokens": observed.cache_write}} + + +def chat_to_anthropic(usage: Any) -> dict: + observed = CacheObservation.from_body({"usage": usage}) + buckets = buckets_from(observed) + return {"input_tokens": buckets["ordinary_input_tokens"], "output_tokens": buckets["output_tokens"], + "cache_creation_input_tokens": buckets["cache_write_input_tokens"], + "cache_read_input_tokens": buckets["cache_read_input_tokens"]} + + +def budget_price(models: list[str], usage: UsageObservation, requests: int) -> float | None: + """Bound a conservative budget estimate, without persisting assumed cache zeros.""" + estimated = CacheObservation(usage.input_tokens, usage.output_tokens, "estimated", ("request_estimate",), + count(usage.input_tokens), 0, 0, "request_estimate") + prices = [price_buckets(model, estimated, requests=requests)["cost_usd"] for model in models] + known = [price for price in prices if price is not None] + return max(known) if known else None diff --git a/services/protocol_translation/anthropic.py b/services/protocol_translation/anthropic.py index 219d34e7..64b0bd58 100644 --- a/services/protocol_translation/anthropic.py +++ b/services/protocol_translation/anthropic.py @@ -850,6 +850,10 @@ def _message_id(value: Any) -> str: def _anthropic_usage_to_chat(usage: Mapping[str, Any]) -> dict: + from services import prompt_cache_cost + + if prompt_cache_cost.enabled(): + return prompt_cache_cost.anthropic_to_chat(dict(usage)) def count(key: str) -> int: value = usage.get(key) return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0 @@ -862,6 +866,10 @@ def count(key: str) -> int: def _chat_usage_to_anthropic(usage: Any) -> dict: + from services import prompt_cache_cost + + if prompt_cache_cost.enabled(): + return prompt_cache_cost.chat_to_anthropic(usage) numbers = usage_numbers(usage) return { "input_tokens": max(0, numbers["prompt"] - numbers["cached"] - numbers["cache_write"]), diff --git a/services/request_accounting.py b/services/request_accounting.py index 50b816ea..01bc2247 100644 --- a/services/request_accounting.py +++ b/services/request_accounting.py @@ -14,14 +14,14 @@ import logging import re import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Iterable, Optional from flask import Response, current_app, g, jsonify, request from werkzeug.exceptions import HTTPException from error_handlers import APIError -from services import key_controls, telemetry_export, usage_ledger +from services import key_controls, prompt_cache_cost, telemetry_export, usage_ledger from services.budget_service import BudgetService, budgeted from services.cost_service import CostService from services.usage_types import StreamUsageObserver, UsageObservation @@ -273,11 +273,17 @@ def _qualified(model: str, provider: Optional[str]) -> str: def _usage_from(value: Any) -> Optional[UsageObservation]: """Input and output tokens from a Chat Completions, Responses or embeddings body.""" - return UsageObservation.from_body(value) + return (prompt_cache_cost.CacheObservation.from_body(value) if prompt_cache_cost.enabled() + else UsageObservation.from_body(value)) + + +def _usage_observer(): + return (prompt_cache_cost.CacheStreamObserver(SSE_TAIL_BYTES) if prompt_cache_cost.enabled() + else StreamUsageObserver(SSE_TAIL_BYTES)) def _sse_usage(tail: bytes) -> Optional[UsageObservation]: - observer = StreamUsageObserver(SSE_TAIL_BYTES) + observer = _usage_observer() observer.feed(tail) return observer.finish() @@ -367,8 +373,10 @@ def _record(context: UsageContext, status: int, usage: Optional[UsageObservation context.input_tokens, context.output_tokens) selected = context.selected or row["selected_model"] models = [selected] if selected else [_qualified(model, context.provider) for model in context.models] - budget_row = {**row, "cost_usd": price(models, estimated.input_tokens, - estimated.output_tokens, context.units)} + estimate = (prompt_cache_cost.budget_price(models, estimated, context.units) + if prompt_cache_cost.enabled() else price(models, estimated.input_tokens, + estimated.output_tokens, context.units)) + budget_row = {**row, "cost_usd": estimate} BudgetService.record_cost(budget_row) usage_ledger.LEDGER.record(row) telemetry_export.EXPORTER.submit({**row, "method": context.method, "trace_id": context.trace[0], @@ -383,7 +391,8 @@ def _record(context: UsageContext, status: int, usage: Optional[UsageObservation def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], units: Optional[int]) -> dict: if usage is not None and context.path == "/v1/embeddings": - usage = UsageObservation(usage.input_tokens, 0, "provider", ("provider",)) + usage = (replace(usage, output_tokens=0) if isinstance(usage, prompt_cache_cost.CacheObservation) + else UsageObservation(usage.input_tokens, 0, "provider", ("provider",))) qualified = [_qualified(model, context.provider) for model in context.models] requested = qualified[0] if len(qualified) == 1 else ("mixed" if qualified else None) selected = context.selected or (requested if requested and requested != "mixed" @@ -396,11 +405,12 @@ def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], elif status < 400 or usage is not None: observation = usage if usage is not None else UsageObservation().with_estimates( context.input_tokens, context.output_tokens) - cost = price([selected] if selected else qualified, observation.input_tokens, - observation.output_tokens, context.units) - basis = observation.storage_basis(cost) + if not (prompt_cache_cost.enabled() and context.kind in {"chat", "responses", "proxy", "embeddings"}): + cost = price([selected] if selected else qualified, observation.input_tokens, + observation.output_tokens, context.units) + basis = observation.storage_basis(cost) prefix = context.user.get("api_key_prefix") - return { + row = { "at": usage_ledger.utc_timestamp(), "principal": str(context.user.get("username") or context.user.get("id") or "unknown")[:256], "key_prefix": str(prefix)[:64] if prefix else None, @@ -416,6 +426,15 @@ def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], "cost_basis": basis, "request_id": context.request_id, } + if prompt_cache_cost.enabled() and context.kind in {"chat", "responses", "proxy", "embeddings"}: + breakdown = CostService.price_buckets(selected or requested, usage, requests=context.units) + row.update({name: breakdown[name] for name in prompt_cache_cost.ROW_FIELDS}) + if context.cached and status < 400: + row.update({name: 0.0 for name in prompt_cache_cost.COST_FIELDS}) + elif status < 400 or usage is not None: + row["cost_usd"] = breakdown["cost_usd"] + row["cost_basis"] = usage.storage_basis(row["cost_usd"]) if usage is not None else None + return row def _capture(context: UsageContext) -> None: @@ -436,7 +455,7 @@ def __init__(self, iterable: Iterable, on_close, event_stream: bool = False) -> self._iterator = iter(iterable) self._on_close = on_close self._tail = b"" - self._usage = StreamUsageObserver(SSE_TAIL_BYTES) if event_stream else None + self._usage = _usage_observer() if event_stream else None def __iter__(self): return self diff --git a/services/usage_store.py b/services/usage_store.py index a1c1a834..b98c134a 100644 --- a/services/usage_store.py +++ b/services/usage_store.py @@ -16,7 +16,7 @@ from contextlib import closing from typing import Any, Optional -from services import intelligence_d1_store +from services import intelligence_d1_store, prompt_cache_cost from services.sqlite_store import connect, storage_path # Upper bounds in milliseconds of the latency buckets; the last bucket is open. @@ -93,6 +93,10 @@ def _checked_rows(value: Any, fields: set[str]) -> list[dict]: return value +def _event_columns() -> tuple: + return EVENT_COLUMNS + (prompt_cache_cost.ROW_FIELDS if prompt_cache_cost.enabled() else ()) + + class D1UsageStore: backend = "d1" @@ -108,6 +112,9 @@ def _call(operation: str, **values) -> dict: return response def record(self, batch_id: str, rows: list[dict]) -> int: + if not prompt_cache_cost.enabled(): + rows = [{name: value for name, value in row.items() if name not in prompt_cache_cost.ROW_FIELDS} + for row in rows] response = self._call("record", batch=batch_id, rows=rows) if set(response) != {"version", "recorded", "duplicate"} or type(response["duplicate"]) is not bool: raise UsageStoreError("invalid record response") @@ -129,7 +136,7 @@ def summary(self, group: str, since: str, until: str, principal: Optional[str], def recent(self, since: str, principal: Optional[str], before: Optional[int], limit: int) -> list[dict]: response = self._call("recent", since=since, principal=principal, before=before, limit=limit) - return _checked_rows(response.get("rows") if set(response) == {"version", "rows"} else None, set(EVENT_COLUMNS)) + return _checked_rows(response.get("rows") if set(response) == {"version", "rows"} else None, set(_event_columns())) def prune(self, events_before: str, rollups_before: str, limit: int) -> dict: response = self._call("prune", events_before=events_before, rollups_before=rollups_before, limit=limit) @@ -159,8 +166,30 @@ def _connect(self): self.ensure(connection) connection.commit() self._ready.add(key) + if prompt_cache_cost.enabled(): + self.ensure_buckets(connection) + connection.commit() return connection + @staticmethod + def ensure_buckets(connection) -> None: + postgres = getattr(connection, "dialect", None) == "postgresql" + if postgres: + columns = {row["name"] for row in connection.execute( + "SELECT column_name AS name FROM information_schema.columns " + "WHERE table_schema = 'multillm' AND table_name = 'usage_events'")} + else: + columns = {row[1] for row in connection.execute("PRAGMA table_info(usage_events)")} + for name in prompt_cache_cost.ROW_FIELDS: + if name not in columns: + kind = "TEXT" if name in {"bucket_basis", "bucket_source"} else "REAL" if name in prompt_cache_cost.COST_FIELDS else "INTEGER" + if postgres and kind == "REAL": + kind = "DOUBLE PRECISION" + elif postgres and kind == "INTEGER": + kind = "BIGINT" + # Names and types come exclusively from fixed first-party constants. + connection.execute(f"ALTER TABLE usage_events ADD COLUMN {name} {kind}") # nosec B608 + @staticmethod def ensure(connection) -> None: connection.execute( @@ -208,6 +237,8 @@ def ensure(connection) -> None: """ ) connection.execute("CREATE INDEX IF NOT EXISTS idx_usage_daily_principal ON usage_daily(principal, day)") + if prompt_cache_cost.enabled(): + SqlUsageStore.ensure_buckets(connection) def record(self, batch_id: str, rows: list[dict]) -> int: if not _BATCH_ID.fullmatch(batch_id): @@ -224,15 +255,16 @@ def record(self, batch_id: str, rows: list[dict]) -> int: totals[5] += int(row["cost_usd"] is not None) totals[6] += row["latency_ms"] totals[7 + latency_bucket(row["latency_ms"])] += 1 + row_fields = ROW_FIELDS + (prompt_cache_cost.ROW_FIELDS if prompt_cache_cost.enabled() else ()) columns = ", ".join(TOTAL_COLUMNS) updates = ", ".join(f"{name} = usage_daily.{name} + excluded.{name}" for name in TOTAL_COLUMNS) try: with closing(self._connect()) as connection: connection.execute("BEGIN IMMEDIATE") connection.executemany( - f"INSERT INTO usage_events (day, {', '.join(ROW_FIELDS)}) " # nosec B608 - f"VALUES ({', '.join('?' for _ in range(len(ROW_FIELDS) + 1))})", - [(row["at"][:10], *(row[name] for name in ROW_FIELDS)) for row in rows], + f"INSERT INTO usage_events (day, {', '.join(row_fields)}) " # nosec B608 + f"VALUES ({', '.join('?' for _ in range(len(row_fields) + 1))})", + [(row["at"][:10], *(row[name] if name in ROW_FIELDS else row.get(name) for name in row_fields)) for row in rows], ) # Column names come only from the fixed TOTAL_COLUMNS definition. connection.executemany( @@ -292,7 +324,7 @@ def recent(self, since: str, principal: Optional[str], before: Optional[int], li parameters.append(before) # Only fixed column names and filters are interpolated. return self._query( - f"SELECT {', '.join(EVENT_COLUMNS)} FROM usage_events WHERE {where} " # nosec B608 + f"SELECT {', '.join(_event_columns())} FROM usage_events WHERE {where} " # nosec B608 "ORDER BY id DESC LIMIT ?", (*parameters, limit), ) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index ff0fe87b..1116edc7 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -345,6 +345,7 @@ "worker/shadow-eval-schedule.mjs", "worker/sse-heartbeat.mjs", "worker/status-page.mjs", + "worker/usage-buckets-d1.mjs", "worker/usage-ledger-d1.mjs" ], "edge_route_literals": { diff --git a/tests/test_control_users_worker.mjs b/tests/test_control_users_worker.mjs index 5922c92d..23321a44 100644 --- a/tests/test_control_users_worker.mjs +++ b/tests/test_control_users_worker.mjs @@ -167,7 +167,7 @@ test("per-key budgets, allowlists, expiry and address ranges persist and are val }); test("accounts keep working when code is deployed before the key-control migration", async () => { - const { mf, call } = await database({ skip: ["0007_usage_ledger.sql"] }); + const { mf, call } = await database({ skip: ["0007_usage_ledger.sql", "0016_usage_buckets.sql"] }); try { assert.equal((await call({ operation: "upsert", user: user("alice") })).status, 200); assert.deepEqual((await call({ operation: "get", username: "alice" })).body.user, user("alice")); diff --git a/tests/test_d1_integration.mjs b/tests/test_d1_integration.mjs index e0319db4..6b419cba 100644 --- a/tests/test_d1_integration.mjs +++ b/tests/test_d1_integration.mjs @@ -70,9 +70,10 @@ test("key controls and the usage ledger persist through the Worker in D1", async }); test("a D1 without the account migration fails closed for dashboard keys but not for the environment admin", async t => { - // 0007, 0011 and 0013 alter control_users, so they are skipped with the migration that creates the table. + // 0007, 0011 and 0013 alter control_users, so they are skipped with the migration that creates the table; + // 0016 alters 0007's usage_events. const { url } = await privateStore(t, ["0003_control_users.sql", "0004_control_user_audit.sql", "0005_auto_routes.sql", - "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql"]); + "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql", "0016_usage_buckets.sql"]); assert.deepEqual(await drive(url, "unmigrated"), { dashboard_key: 503, admin: "admin", route: ["gguu:gpt-image-2.5-sunburst", "gguu:gpt-image-2.5"], save_route: 503 }); }); diff --git a/tests/test_d1_schema_worker.mjs b/tests/test_d1_schema_worker.mjs index 455ef284..44701153 100644 --- a/tests/test_d1_schema_worker.mjs +++ b/tests/test_d1_schema_worker.mjs @@ -29,8 +29,8 @@ test("/ready reports a D1 schema that was never migrated instead of forwarding", const worker = (await loadWorkerModule()).default; let forwarded = 0; const container = { getByName: () => ({ fetch: async () => { forwarded += 1; return Response.json({ status: "healthy" }); } }) }; - // 0007 alters control_users, so skipping 0003 skips it too. - const partial = await database(t, { skip: ["0003_control_users.sql", "0005_auto_routes.sql", "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql"] }); + // 0007 alters control_users, so skipping 0003 skips it too; 0016 alters 0007's usage_events. + const partial = await database(t, { skip: ["0003_control_users.sql", "0005_auto_routes.sql", "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql", "0016_usage_buckets.sql"] }); assert.deepEqual(await d1Readiness(partial), { ready: false, checked: true, missing: ["control_users", "auto_routes", "usage_events", "usage_daily", "usage_batches", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups"] }); const blocked = await worker.fetch(new Request("https://gateway.example/ready"), { INTELLIGENCE_DB: partial, MULTILLM_PROXY_CONTAINER: container }); assert.equal(blocked.status, 503); diff --git a/tests/test_prompt_cache_cost.py b/tests/test_prompt_cache_cost.py new file mode 100644 index 00000000..12e53e99 --- /dev/null +++ b/tests/test_prompt_cache_cost.py @@ -0,0 +1,270 @@ +import json +import sqlite3 +from pathlib import Path + +import pytest +from flask import Flask, Response, g + +from services import prompt_cache_cost as cache, request_accounting as accounting, usage_store +from services.cost_service import CostService +from services.protocol_translation.anthropic import _anthropic_usage_to_chat, _chat_usage_to_anthropic +from tests.test_usage_ledger import row + + +@pytest.fixture(autouse=True) +def settings(monkeypatch): + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', 'true') + monkeypatch.setenv('CONTROL_PLANE_DATABASE_URL', '') + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'openai:m': { + 'input': 2, 'output': 8, 'cache_read': 0.5, 'cache_write': 3}})) + monkeypatch.setenv('PROMPT_CACHE_PRICE_METADATA_JSON', '{}') + + +def observe(usage): + return accounting._usage_from({'usage': usage}) + + +def context(model='openai:m'): + return accounting.UsageContext(kind='chat', models=[model], provider=None, + user={'username': 'reader'}, started=accounting.time.perf_counter(), start_ns=1, + input_tokens=50, output_tokens=60, path='/v1/chat/completions', trace=(None, None)) + + +@pytest.mark.parametrize('flag', ['', 'false', 'garbage']) +def test_disabled_preserves_rows_prices_translation_and_schema(monkeypatch, flag): + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', flag) + usage = observe({'prompt_tokens': 100, 'completion_tokens': 10, + 'prompt_tokens_details': {'cached_tokens': 60}}) + result = accounting._row(context(), 200, usage, None) + assert set(result) == set(usage_store.ROW_FIELDS) + assert result['cost_usd'] == 0.00028 + assert _anthropic_usage_to_chat({})['prompt_tokens'] == 0 + connection = sqlite3.connect(':memory:') + usage_store.SqlUsageStore.ensure(connection) + assert 'ordinary_input_tokens' not in {item[1] for item in connection.execute('PRAGMA table_info(usage_events)')} + + +@pytest.mark.parametrize('usage,expected', [ + ({'prompt_tokens': 100, 'completion_tokens': 10, 'prompt_tokens_details': {'cached_tokens': 60}}, (40, 60, 0, 10)), + ({'input_tokens': 40, 'output_tokens': 10, 'cache_read_input_tokens': 60, 'cache_creation_input_tokens': 20}, (40, 60, 20, 10)), + ({'prompt_tokens': 100, 'completion_tokens': 0, 'prompt_tokens_details': {'cached_tokens': 0}}, (100, 0, 0, 0)), + ({'prompt_tokens': 100, 'completion_tokens': 10}, (None, None, 0, 10)), + ({'prompt_tokens': 4, 'prompt_tokens_details': {'cached_tokens': 8}}, (None, 8, 0, None)), + ({'input_tokens': True, 'cache_creation_input_tokens': -1}, (None, None, None, None)), +]) +def test_inclusive_exclusive_and_unknown_counts(usage, expected): + buckets = cache.buckets_from(observe(usage)) + assert tuple(buckets[name] for name in cache.TOKEN_FIELDS) == expected + + +def test_costs_and_metadata_supplement_only(monkeypatch): + usage = observe({'prompt_tokens': 100, 'completion_tokens': 10, + 'prompt_tokens_details': {'cached_tokens': 60}}) + result = accounting._row(context(), 200, usage, None) + assert result['cost_usd'] == 0.00019 + assert result['ordinary_input_cost_microusd'] == 80 + assert result['cache_read_input_cost_microusd'] == 30 + assert result['bucket_basis'] == 'measured' + monkeypatch.setenv('PROMPT_CACHE_PRICE_METADATA_JSON', json.dumps({'openai:m': {'cache_read': 0.01}})) + assert CostService.price_buckets('openai:m', usage)['cost_usd'] == 0.00019 + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'openai:m': {'input': 2, 'output': 8}})) + assert CostService.price_buckets('openai:m', usage)['cost_usd'] == 0.0001606 + assert CostService.price_buckets('missing', usage)['cost_usd'] is None + monkeypatch.setenv('PROMPT_CACHE_PRICE_METADATA_JSON', '{}') + missing = CostService.price_buckets('openai:m', usage) + assert missing['cost_usd'] is None and missing['ordinary_input_cost_microusd'] == 80 + assert missing['cache_read_input_cost_microusd'] is None + + +def test_missing_counts_zero_prices_and_bounds(monkeypatch): + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'free': { + 'input': 0, 'output': 0, 'cache_read': 0, 'cache_write': 0}})) + assert CostService.price_buckets('free', observe({}))['cost_usd'] == 0 + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'huge': { + 'input': '1e100', 'output': 0, 'cache_read': 0, 'cache_write': 0}})) + assert CostService.price_buckets('huge', observe({'prompt_tokens': 2, 'prompt_tokens_details': {'cached_tokens': 0}}))['cost_usd'] is None + assert cache.buckets_from(observe({'prompt_tokens': 2**53}))['ordinary_input_tokens'] is None + + +def test_split_sse_and_translations_preserve_unknown_and_cache_counts(): + observer = cache.CacheStreamObserver(256) + data = (b'data: {"message":{"usage":{"input_tokens":40,"cache_read_input_tokens":60,"cache_creation_input_tokens":20}}}\n\n' + b'data: {"usage":{"output_tokens":10}}\n\n') + for start in range(0, len(data), 7): + observer.feed(data[start:start+7]) + usage = observer.finish() + assert tuple(cache.buckets_from(usage)[name] for name in cache.TOKEN_FIELDS) == (40, 60, 20, 10) + chat = _anthropic_usage_to_chat({'input_tokens': 40, 'cache_read_input_tokens': 60, 'cache_creation_input_tokens': 20}) + assert chat['prompt_tokens'] == 120 and chat['completion_tokens'] is None + reverse = _chat_usage_to_anthropic(chat) + assert reverse == {'input_tokens': 40, 'output_tokens': None, 'cache_creation_input_tokens': 20, 'cache_read_input_tokens': 60} + assert _anthropic_usage_to_chat({})['prompt_tokens'] is None + + +def test_registered_accounting_stream_keeps_body_and_records_once(monkeypatch): + app = Flask(__name__) + rows = [] + monkeypatch.setattr(accounting.usage_ledger.LEDGER, 'record', lambda row: rows.append(row)) + monkeypatch.setattr(accounting.BudgetService, 'record_cost', lambda row: None) + monkeypatch.setattr(accounting.BudgetService, 'settle', lambda reservation: None) + monkeypatch.setattr(accounting.telemetry_export.EXPORTER, 'submit', lambda row: None) + data = b'data: {"usage":{"prompt_tokens":100,"completion_tokens":10,"prompt_tokens_details":{"cached_tokens":60}}}\n\n' + @app.post('/v1/chat/completions') + def completion(): + g.usage_context = context() + return accounting.finish(Response((data[i:i+9] for i in range(0, len(data), 9)), mimetype='text/event-stream')) + response = app.test_client().post('/v1/chat/completions') + assert response.get_data() == data + response.close() + assert len(rows) == 1 and rows[0]['cache_read_input_tokens'] == 60 + + +def test_sqlite_additive_migration_old_rows_and_bucket_serialization(tmp_path, monkeypatch): + monkeypatch.setenv('USAGE_DB_PATH', str(tmp_path/'usage.sqlite3')) + store = usage_store.SqlUsageStore() + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', 'false') + store.record('a'*32, [row()]) + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', 'true') + result = accounting._row(context(), 200, observe({'prompt_tokens': 100, 'completion_tokens': 10, + 'prompt_tokens_details': {'cached_tokens': 60}}), None) + store.record('b'*32, [result]) + recent = store.recent('2000', None, None, 10) + assert recent[0]['cache_read_input_tokens'] == 60 + assert recent[1]['cache_read_input_tokens'] is None + db = sqlite3.connect(':memory:') + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', 'false') + store.ensure(db) + db.executescript(Path('intelligence-migrations/0016_usage_buckets.sql').read_text()) + assert set(cache.ROW_FIELDS) <= {item[1] for item in db.execute('PRAGMA table_info(usage_events)')} + + +def test_d1_enabled_sends_nullable_buckets_without_changing_disabled_payload(monkeypatch): + calls = [] + monkeypatch.setattr(usage_store.intelligence_d1_store, 'request_private_intelligence', + lambda body, **kwargs: calls.append(body) or {'version': 1, 'recorded': 1, 'duplicate': False}) + result = accounting._row(context(), 200, observe({}), None) + usage_store.D1UsageStore().record('a'*32, [result]) + assert calls[0]['rows'][0]['cache_read_input_tokens'] is None + monkeypatch.setenv('PROMPT_CACHE_USAGE_BUCKETS_ENABLED', 'false') + usage_store.D1UsageStore().record('b'*32, [result]) + assert set(calls[1]['rows'][0]) == set(usage_store.ROW_FIELDS) + + +@pytest.mark.parametrize('metadata', ['bad json', '[]', '{"openai:m":{"cache_read":-1}}']) +def test_bad_metadata_disables_feature_without_logging_value(monkeypatch, caplog, metadata): + monkeypatch.setenv('PROMPT_CACHE_PRICE_METADATA_JSON', metadata) + assert not cache.enabled() + assert metadata not in caplog.text + assert set(accounting._row(context(), 200, observe({'prompt_tokens': 1}), None)) == set(usage_store.ROW_FIELDS) + + +def test_stream_keeps_prior_counts_after_nulls_and_drops_oversized_lines(): + observer = cache.CacheStreamObserver(200) + observer.feed(b'data: {"usage":{"prompt_tokens":100,"prompt_tokens_details":{"cached_tokens":60},"completion_tokens":0}}\n') + observer.feed(b'data: ' + b'x'*201 + b'\n') + observer.feed(b'data: {"usage":{"prompt_tokens":null,"completion_tokens":null}}\n') + buckets = cache.buckets_from(observer.finish()) + assert tuple(buckets[name] for name in cache.TOKEN_FIELDS) == (40, 60, 0, 0) + + +def test_sqlite_schema_upgrade_and_writes_are_safe_under_concurrency(tmp_path, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + + monkeypatch.setenv('USAGE_DB_PATH', str(tmp_path/'parallel.sqlite3')) + with ThreadPoolExecutor(max_workers=4) as pool: + assert list(pool.map(lambda i: usage_store.SqlUsageStore().record(format(i, '032x'), [row()]), range(8))) == [1]*8 + assert len(usage_store.SqlUsageStore().recent('2000', None, None, 20)) == 8 + + +def test_real_authenticated_route_keeps_native_body_and_bucket_ledger(tmp_path, monkeypatch): + from unittest.mock import patch + from services import usage_ledger + from tests.unified_api_test_case import UnifiedApiTestCase + + monkeypatch.setenv('USAGE_LEDGER_ENABLED', 'true') + monkeypatch.setenv('USAGE_DB_PATH', str(tmp_path/'routes.sqlite3')) + monkeypatch.setenv('USAGE_LEDGER_BACKEND', 'sql') + monkeypatch.setenv('USAGE_LEDGER_FLUSH_SECONDS', '300') + fixture = UnifiedApiTestCase() + usage_ledger.LEDGER.reset() + with patch('config.load_runtime_env'): + fixture.setUp() + try: + upstream = fixture._chat_response() + body = {'id': 'chatcmpl-cache', 'object': 'chat.completion', + 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': 'ok'}, 'finish_reason': 'stop'}], + 'usage': {'prompt_tokens': 100, 'completion_tokens': 10, 'prompt_tokens_details': {'cached_tokens': 60}}} + upstream._content = json.dumps(body).encode() + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'opencode:m': {'input': 2, 'output': 8, 'cache_read': 0.5}})) + with patch('app.ProxyService.make_request', return_value=upstream): + response = fixture.client.post('/v1/chat/completions', headers={'Authorization': 'Bearer admin-test-key'}, + json={'model': 'opencode:m', 'messages': [{'role': 'user', 'content': 'hi'}]}) + assert response.status_code == 200 + assert response.get_json()['usage'] == body['usage'] + assert usage_ledger.LEDGER.flush(timeout=5) + stored = usage_ledger.LEDGER.store().recent('2000', None, None, 10) + assert stored[0]['cache_read_input_tokens'] == 60 and stored[0]['cost_usd'] == 0.00019 + finally: + usage_ledger.LEDGER.reset() + fixture.tearDown() + + +def test_enabled_accounting_does_not_run_unbounded_legacy_costs(monkeypatch): + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'openai:m': { + 'input': '1e100', 'output': 0, 'cache_read': 0, 'cache_write': 0}})) + result = accounting._row(context(), 200, observe({'prompt_tokens': 2, 'completion_tokens': 0, + 'prompt_tokens_details': {'cached_tokens': 0}}), None) + assert result['cost_usd'] is None and result['ordinary_input_tokens'] == 2 + + +def test_python_worker_serialization_parity(): + import subprocess + + result = accounting._row(context(), 200, observe({'prompt_tokens': 100, 'completion_tokens': 10, + 'prompt_tokens_details': {'cached_tokens': 60}}), None) + process = subprocess.run(['node', '--input-type=module', '-e', + "import {serializeUsageBuckets} from './worker/usage-buckets-d1.mjs';" + "let s='';for await(const b of process.stdin)s+=b;" + "console.log(JSON.stringify(serializeUsageBuckets(JSON.parse(s),{PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true'})));"], + input=json.dumps(result), text=True, capture_output=True, check=True) + assert json.loads(process.stdout) == result + + +def test_partial_cache_budget_estimates_and_failure_recording_are_bounded(monkeypatch): + from services.usage_types import UsageObservation + + estimated = UsageObservation().with_estimates(100, 10) + assert cache.buckets_from(estimated)['bucket_basis'] == 'estimated' + assert cache.budget_price(['openai:m'], estimated, 1) == 0.00028 + monkeypatch.setenv('MODEL_PRICING_USD_PER_MILLION', json.dumps({'openai:m': {'input': '1e100', 'output': 0}})) + rows = [] + monkeypatch.setattr(accounting.usage_ledger.LEDGER, 'record', lambda row: rows.append(row)) + monkeypatch.setattr(accounting.BudgetService, 'record_cost', lambda row: None) + monkeypatch.setattr(accounting.BudgetService, 'settle', lambda reservation: None) + monkeypatch.setattr(accounting.telemetry_export.EXPORTER, 'submit', lambda row: None) + accounting._record(context(), 200, observe({'prompt_tokens': 2, 'completion_tokens': 0}), None) + assert len(rows) == 1 and rows[0]['cost_usd'] is None + + +def test_embeddings_keep_cache_observations_with_known_zero_output(): + c = context() + c.kind, c.path = 'embeddings', '/v1/embeddings' + result = accounting._row(c, 200, observe({'prompt_tokens': 100, + 'prompt_tokens_details': {'cached_tokens': 60}}), None) + assert result['cache_read_input_tokens'] == 60 + assert result['ordinary_input_tokens'] == 40 + assert result['output_tokens'] == 0 and result['cost_usd'] == 0.00011 + + +def test_migration_adds_nullable_bucket_columns_and_keeps_existing_rows(): + db = sqlite3.connect(':memory:') + for path in sorted(Path('intelligence-migrations').glob('*.sql')): + if path.name.startswith('0016_'): + db.execute("INSERT INTO usage_events (at, day, principal, kind, endpoint, status, latency_ms) " + "VALUES ('2026-10-09T00:00:00Z', '2026-10-09', 'old', 'chat', '/v1/chat/completions', 200, 5)") + db.executescript(path.read_text()) + columns = {item[1] for item in db.execute('PRAGMA table_info(usage_events)')} + assert set(usage_store.EVENT_COLUMNS) <= columns + assert set(cache.ROW_FIELDS) <= columns + assert db.execute('SELECT principal, ordinary_input_tokens, bucket_basis FROM usage_events').fetchall() == [ + ('old', None, None)] diff --git a/tests/test_prompt_cache_cost_worker.mjs b/tests/test_prompt_cache_cost_worker.mjs new file mode 100644 index 00000000..62c84168 --- /dev/null +++ b/tests/test_prompt_cache_cost_worker.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { readFile } from 'node:fs/promises'; +import { convertV4MiniflareOptions, Miniflare } from 'miniflare'; +import { BUCKET_FIELDS, serializeUsageBuckets, recordUsageWithBuckets, bucketInsertStatement } from '../worker/usage-buckets-d1.mjs'; +import { validRow } from '../worker/usage-ledger-d1.mjs'; +import { applyMigrations } from './d1_migrations.mjs'; + +const base = {at:'2026-10-01T00:00:00.000Z', principal:'reader', key_prefix:null, kind:'chat', + endpoint:'/v1/chat/completions', requested_model:'openai:m', selected_model:'openai:m', status:200, + latency_ms:1, input_tokens:100, output_tokens:10, cost_usd:0.00019, cost_basis:'usage', request_id:null}; +const buckets = Object.fromEntries(BUCKET_FIELDS.map(name => [name, null])); +Object.assign(buckets, {ordinary_input_tokens:40, cache_read_input_tokens:60, cache_write_input_tokens:0, + ordinary_input_cost_microusd:80, cache_read_input_cost_microusd:30, cache_write_input_cost_microusd:0, + output_cost_microusd:80, bucket_basis:'measured', bucket_source:'openai'}); +const row = {...base, ...buckets}; + +test('disabled serializer returns legacy row and does not execute new statements', async () => { + assert.deepEqual(serializeUsageBuckets(row, {}), base); + assert.ok(validRow(serializeUsageBuckets(row, {}))); + let extended = 0; + const result = await recordUsageWithBuckets({}, {rows:[row]}, { + recordBase: async body => {assert.deepEqual(body.rows,[base]); return {version:1,recorded:1,duplicate:false};}, + recordExtended: async () => {extended++;}, + }); + assert.equal(result.status,200); assert.equal(extended,0); + for (const flag of ['', 'false', 'malformed']) assert.deepEqual(serializeUsageBuckets(row,{PROMPT_CACHE_USAGE_BUCKETS_ENABLED:flag}),base); +}); + +test('strict nullable counts, bounded microUSD and provenance', () => { + const env={PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true'}; + assert.deepEqual(serializeUsageBuckets(row,env),row); + for (const value of [-1, true, '3', Number.MAX_SAFE_INTEGER+1]) + assert.throws(()=>serializeUsageBuckets({...row,cache_read_input_tokens:value},env)); + assert.throws(()=>serializeUsageBuckets({...row,output_cost_microusd:1e12+1},env)); + assert.throws(()=>serializeUsageBuckets({...row,bucket_source:'secret text'},env)); + assert.equal(serializeUsageBuckets({...row,cache_read_input_tokens:null},env).cache_read_input_tokens,null); + assert.throws(()=>serializeUsageBuckets({...row,unexpected:1},env)); +}); + +test('migration accepts old rows; missing columns fail closed after preserving base ledger', async t => { + const mf=new Miniflare(convertV4MiniflareOptions({modules:true,script:"export default {fetch(){return new Response('ok')}}",d1Databases:['DB']})); + t.after(()=>mf.dispose()); const db=await mf.getD1Database('DB'); + await applyMigrations(db,{skip:['0016_usage_buckets.sql']}); + const legacySql='INSERT INTO usage_events (day,'+Object.keys(base).join(',')+') VALUES ('+Array(Object.keys(base).length+1).fill('?').join(',')+')'; + const callbacks={recordBase:async body=>{await db.prepare(legacySql).bind('2026-10-01',...Object.values(body.rows[0])).run();return {version:1,recorded:1,duplicate:false};}, + recordExtended:async body=>{await db.prepare(bucketInsertStatement(Object.keys(base))).bind(JSON.stringify(body.rows),'a'.repeat(32),'token').run(); return {version:1,recorded:1,duplicate:false};}}; + await db.prepare('INSERT INTO usage_batches(id,token,at) VALUES (?,?,?)').bind('a'.repeat(32),'token',base.at).run(); + const response=await recordUsageWithBuckets({PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true'},{rows:[row]},callbacks); + assert.equal(response.status,503); assert.equal((await response.json()).error.code,'usage_buckets_unavailable'); + assert.equal((await db.prepare('SELECT COUNT(*) AS n FROM usage_events').first()).n,1); + const sql=await readFile(new URL('../intelligence-migrations/0016_usage_buckets.sql',import.meta.url),'utf8'); + for(const statement of sql.replace(/^--.*$/gm,'').split(';').filter(s=>s.trim())) await db.prepare(statement).run(); + const success=await recordUsageWithBuckets({PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true'},{rows:[row]},callbacks); + assert.equal(success.status,200); + const rows=(await db.prepare('SELECT * FROM usage_events ORDER BY id').all()).results; + assert.equal(rows[0].cache_read_input_tokens,null); assert.equal(rows[1].cache_read_input_tokens,60); +}); + + +test('bad metadata and missing base storage stay closed without leaking details', async () => { + for(const metadata of ['[]', 'bad json', '{"m":{"cache_read":-1}}', '{"m":{"cache_read":"0x10"}}']) { + assert.deepEqual(serializeUsageBuckets(row,{PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true',PROMPT_CACHE_PRICE_METADATA_JSON:metadata}),base); + } + const response=await recordUsageWithBuckets({PROMPT_CACHE_USAGE_BUCKETS_ENABLED:'true'},{rows:[row]}, { + recordBase: async ()=>{throw new Error('private detail');}, recordExtended: async ()=>{throw new Error('private detail');}, + }); + assert.equal(response.status,503); assert.equal((await response.json()).error.code,'storage_unavailable'); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index f678771b..9b20c4eb 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -51,6 +51,8 @@ const DIRECT_ENV_KEYS = [ "CONTEXT_ANALYSIS_CACHE_TTL_SECONDS", "CONTEXT_ANALYSIS_CACHE_MAX_ENTRIES", "PROMPT_CACHE_ENABLED", + "PROMPT_CACHE_USAGE_BUCKETS_ENABLED", + "PROMPT_CACHE_PRICE_METADATA_JSON", "TOOL_CALL_REPAIR_DEFAULT", "PROMPT_CACHE_MIN_TOKENS", "PROMPT_TEMPLATES_ENABLED", diff --git a/worker/usage-buckets-d1.mjs b/worker/usage-buckets-d1.mjs new file mode 100644 index 00000000..8d0f1c8e --- /dev/null +++ b/worker/usage-buckets-d1.mjs @@ -0,0 +1,82 @@ +/** Opt-in strict bucket serialization; the ledger owner injects transaction writers. */ +import { validRow } from "./usage-ledger-d1.mjs"; + +export const BUCKET_FIELDS = Object.freeze(["ordinary_input_tokens", "cache_read_input_tokens", "cache_write_input_tokens", + "ordinary_input_cost_microusd", "cache_read_input_cost_microusd", "cache_write_input_cost_microusd", "output_cost_microusd", + "bucket_basis", "bucket_source"]); +const BASE_FIELDS = Object.freeze(["at", "principal", "key_prefix", "kind", "endpoint", "requested_model", "selected_model", + "status", "latency_ms", "input_tokens", "output_tokens", "cost_usd", "cost_basis", "request_id"]); +const warned = new Set(); +const warn = name => { if (!warned.has(name)) { warned.add(name); console.warn(`Invalid ${name}; prompt cache buckets disabled`); } }; +export function usageBucketsEnabled(env) { + const flag = String(env.PROMPT_CACHE_USAGE_BUCKETS_ENABLED ?? "").trim().toLowerCase(); + if (["", "0", "false", "no", "off"].includes(flag)) return false; + if (!["1", "true", "yes", "on"].includes(flag)) { warn("PROMPT_CACHE_USAGE_BUCKETS_ENABLED"); return false; } + const raw = String(env.PROMPT_CACHE_PRICE_METADATA_JSON ?? "").trim(); + try { + const parsed = JSON.parse(raw || "{}"); + if (raw.length > 262144 || !parsed || Array.isArray(parsed) || typeof parsed !== "object" + || Object.values(parsed).some(value => !value || Array.isArray(value) || typeof value !== "object" + || Object.values(value).some(price => !["number", "string"].includes(typeof price) + || !/^[+\-]?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+\-]?\d+)?$/i.test(String(price).trim()) + || !Number.isFinite(Number(price)) || Number(price) < 0 || Number(price) > 1e12))) throw new Error(); + } catch { warn("PROMPT_CACHE_PRICE_METADATA_JSON"); return false; } + return true; +} + +export function serializeUsageBuckets(row, env) { + if (!row || typeof row !== "object" || Array.isArray(row)) throw new TypeError("invalid usage row"); + const base = Object.fromEntries(BASE_FIELDS.map(name => [name, row[name]])); + if (!validRow(base)) throw new TypeError("invalid usage row"); + const allowed = new Set([...BASE_FIELDS, ...BUCKET_FIELDS]); + if (Object.keys(row).some(name => !allowed.has(name))) throw new TypeError("invalid usage fields"); + if (!usageBucketsEnabled(env)) return base; + const result = { ...base }; + for (const name of BUCKET_FIELDS) { + const value = row[name] ?? null; + let valid = value === null; + if (name.endsWith("tokens")) valid ||= Number.isSafeInteger(value) && value >= 0; + else if (name.endsWith("microusd")) valid ||= typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1e12; + else if (name === "bucket_basis") valid ||= ["measured", "estimated", "unknown"].includes(value); + else valid ||= ["openai", "anthropic", "request_estimate", "unknown"].includes(value); + if (!valid) throw new TypeError("invalid usage bucket"); + result[name] = value; + } + return result; +} + +/** Fixed JSON insertion replaces INSERT_EVENTS inside the existing batch transaction. */ +export function bucketInsertStatement(baseFields = BASE_FIELDS) { + if (baseFields.length !== BASE_FIELDS.length || baseFields.some((name, i) => name !== BASE_FIELDS[i])) { + throw new TypeError("invalid ledger columns"); + } + const names = [...BASE_FIELDS, ...BUCKET_FIELDS]; + return `INSERT INTO usage_events (day, ${names.join(", ")}) + SELECT substr(json_extract(value, '$.at'), 1, 10), ${names.map(name => `json_extract(value, '$.${name}')`).join(", ")} + FROM json_each(?1) WHERE EXISTS (SELECT 1 FROM usage_batches WHERE id = ?2 AND token = ?3)`; +} + +/** Extended writes must be atomic/idempotent; on failure the legacy writer saves the base row. */ +export async function recordUsageWithBuckets(env, body, { recordBase, recordExtended }) { + const enabled = usageBucketsEnabled(env); + let rows; + try { + if (!Array.isArray(body.rows) || body.rows.length < 1 || body.rows.length > 500) throw new TypeError(); + rows = body.rows.map(row => serializeUsageBuckets(row, env)); + } catch { + return Response.json({ version: 1, error: { code: "invalid_request", message: "Usage ledger operation failed" } }, { status: 400 }); + } + const legacy = { ...body, rows: rows.map(row => Object.fromEntries(BASE_FIELDS.map(name => [name, row[name]]))) }; + const headers = { "cache-control": "no-store" }; + if (!enabled) return Response.json(await recordBase(legacy), { headers }); + try { + return Response.json(await recordExtended({ ...body, rows }), { headers }); + } catch { + try { await recordBase(legacy); } catch { + return Response.json({ version: 1, error: { code: "storage_unavailable", message: "Usage ledger operation failed" } }, + { status: 503, headers }); + } + return Response.json({ version: 1, error: { code: "usage_buckets_unavailable", + message: "Prompt cache usage storage unavailable; verify migration 0016" } }, { status: 503, headers }); + } +}