From ab5be971e16c7a1d3ef5802e9e478ff72f26b839 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 06:56:51 +0530 Subject: [PATCH] feat(observability): add opt-in Langfuse and Helicone exports Send content-free request observations to explicitly configured Langfuse and Helicone collectors when OBSERVABILITY_EXPORTERS_JSON lists them. Delivery uses a bounded in-memory queue with retries limited to transport failures, 429 and 5xx, and counts acknowledgements per exporter. Credentials come from variables ending in _EXPORT_CREDENTIAL. With the setting unset, nothing is sent and OTLP export is unchanged. --- docs/observability-exports.md | 103 +++++++ package.json | 2 +- services/observability_adapters.py | 308 +++++++++++++++++++ services/telemetry_export.py | 6 + tests/fixtures/secret_firewall_coverage.json | 1 + tests/test_observability_exports.py | 243 +++++++++++++++ tests/test_observability_exports_worker.mjs | 158 ++++++++++ worker/container-env.mjs | 3 + worker/observability-export.mjs | 253 +++++++++++++++ worker/usage-ledger-d1.mjs | 4 +- 10 files changed, 1079 insertions(+), 2 deletions(-) create mode 100644 docs/observability-exports.md create mode 100644 services/observability_adapters.py create mode 100644 tests/test_observability_exports.py create mode 100644 tests/test_observability_exports_worker.mjs create mode 100644 worker/observability-export.mjs diff --git a/docs/observability-exports.md b/docs/observability-exports.md new file mode 100644 index 00000000..91eef435 --- /dev/null +++ b/docs/observability-exports.md @@ -0,0 +1,103 @@ +# External observability exports + +The gateway can send content-free request observations to explicit Langfuse and +Helicone collectors. `OBSERVABILITY_EXPORTERS_JSON` defaults to `[]`, which sends +nothing. An unset or empty value also disables export. Malformed JSON, unsupported +exporter types, unknown fields or invalid destinations disable all exporters and +produce one warning without the setting's value. + +Configure each destination with exactly these fields: + +```json +[ + { + "type": "langfuse", + "endpoint": "https://langfuse.example/api/public/ingestion", + "allowed_origins": ["https://langfuse.example"], + "credential_env": "LANGFUSE_EXPORT_CREDENTIAL" + }, + { + "type": "helicone", + "endpoint": "https://helicone.example/custom/v1/log", + "allowed_origins": ["https://helicone.example"], + "credential_env": "HELICONE_EXPORT_CREDENTIAL" + } +] +``` + +Replace the example destinations with the collector ingestion endpoints you +operate. There are no implicit hosts or fallback destinations. Endpoints must use +HTTPS and match an exact configured origin, including a non-default port. Wildcards, +URL credentials, query strings, fragments and redirects are refused. Origin entries +have no path beyond `/`. Configuration is limited to 32 KiB, eight exporters and +sixteen origins per exporter. Duplicate destinations with the same type and +credential reference are invalid. + +`credential_env` names a private environment variable whose name ends in +`_EXPORT_CREDENTIAL`, so a provider or administrator secret cannot be sent to a +collector by mistake. For Langfuse its value is +the public key and secret key joined by a colon; it is encoded as HTTP Basic +authentication at delivery time. For Helicone its value is the collector token, +sent as HTTP Bearer authentication. Values appear only in outbound authentication +headers. They never appear in configuration examples, export bodies, status or logs. +An absent or empty credential causes a delivery failure without an HTTP call. +The Flask Container must receive the configured credential variables through its +explicit environment allowlist. Configure Worker bindings separately. + +Observations contain request and correlation IDs, a hashed principal identifier, +the selected authorized model, request kind, status, a fixed error classification, +timings, nullable token counts, nullable cost, and usage/cost provenance. They never +include prompts, generated output, tools, API-key prefixes, raw request or response +headers, arbitrary metadata or upstream error messages. Model names come from +trusted finalization metadata. Zero token counts and zero costs remain distinct +from unknown values. Usage provenance remains `unknown` when unavailable; no price +is inferred from a collector response. Shadow and canary observations retain their +own kind and cost instead of being folded into production request totals. + +Langfuse receives `generation-create` ingestion events with stable IDs and the +observation in generation metadata. Helicone receives content-free custom log +envelopes, split seconds/milliseconds timing, and the observation as the +`Helicone-Property-observation` property. The structural provider URL is the neutral +`https://multillm.invalid`; it is never used for a generation or network request. +Collectors requiring prompt/output fields cannot use this export mode. There is +no content-capture option. + +Each process or Worker environment has an in-memory queue of at most 1,000 +observations. Delivery takes at most 50 observations per batch; Helicone's custom +endpoint uses one HTTP envelope per observation. Every delivery attempt has a +two-second timeout and at most two retries, limited to transport failures, HTTP +429 and HTTP 5xx. Redirects and other HTTP errors are terminal. Export retries +never invoke a generation or change its retry permission. Stable event IDs are +reused on collector retries, but remote duplicate suppression depends on the +collector. The queue holds no durable telemetry: restarts or Worker eviction can +lose accepted records. No D1 or R2 table is created for exports. + +Flask submissions only enqueue; a daemon thread delivers them independently of +OTLP. The existing OTLP payloads, headers, queue, return values and configuration +remain unchanged with exporters enabled or disabled. Native Worker finalization +enqueues its own observation and schedules export with the request execution +context's `waitUntil`. Forwarded requests export from Flask; their private D1 +ledger flush never exports again. Worker collection and scheduling require the +native finalization hook to supply that execution context. Without scheduling, +acceptance only buffers the observation until an explicit flush. + +`OBSERVABILITY_EXPORTER.status()` in Flask and +`getObservabilityExporter(env).status()` in the Worker return accepted, queued and +dropped counts. Each exporter reports its type, credential variable name, +attempt count, acknowledged count, failed count and delivery status (`idle`, +`acknowledged`, `partial`, `failed` or `credential_unavailable`). These are internal +operator interfaces, not new public endpoints. Queue acceptance is **accepted**, +never delivered. A 2xx HTTP acknowledgement indicates collector acceptance, not +proof of persistent storage. Langfuse HTTP 207 receipts count only successful +event IDs belonging to the submitted batch and exclude IDs listed as errors; +missing, malformed or oversized receipts acknowledge nothing. Partial receipts +are not retried. Collector response bodies and exception details are never logged. + +Delivery failures are independent across exporters and do not alter generation +responses, accounting, storage or authorization. The queue drops newest items +when full and counts those drops. Deduplication retains the most recent 1,000 +accepted request/owner/kind identities; it is process-local and does not promise +distributed exactly-once delivery. Content-free export is compatible with zero +content retention, but identifiers, model names, timings and usage still leave +the gateway at the operator's explicit direction. Local tests use fake collectors; +they do not establish live collector compatibility, persistence or deployment. diff --git a/package.json b/package.json index e90d16c7..4153f729 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_observability_exports_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/services/observability_adapters.py b/services/observability_adapters.py new file mode 100644 index 00000000..aace0b8f --- /dev/null +++ b/services/observability_adapters.py @@ -0,0 +1,308 @@ +"""Explicit content-free collectors, independent of the OTLP exporter.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import math +import os +import re +import threading +from collections import deque +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Any, Mapping +from urllib.parse import urlsplit + +import requests + +logger = logging.getLogger(__name__) +MAX_QUEUE = 1000 +BATCH_SIZE = 50 +TIMEOUT = 2 +RETRIES = 2 +_warned = False +_KINDS = {"chat", "responses", "embeddings", "images", "audio", "proxy", "shadow", "canary", "roleplay"} +_OUTCOMES = {"success", "unknown", "upstream_error", "transport_error", "canceled"} +_BASES = {"usage", "estimate", "cache"} +_PROVENANCE = {"provider", "request_estimate", "cache", "measured", "estimated", "unknown"} + + +@dataclass(frozen=True) +class Destination: + type: str + endpoint: str + credential_env: str + + +def _origin(value: str, *, origin_only: bool = False) -> str: + if not isinstance(value, str) or len(value) > 2048 or re.search(r"[\s\x00-\x1f\x7f\\]", value): + raise ValueError("Invalid destination") + url = urlsplit(value) + if (url.scheme != "https" or not url.hostname or url.username is not None or url.password is not None + or "?" in value or "#" in value or origin_only and url.path not in {"", "/"}): + raise ValueError("Invalid destination") + port = url.port + host = url.hostname.lower() + host = f"[{host}]" if ":" in host else host + return f"https://{host}" + (f":{port}" if port not in {None, 443} else "") + + +def destinations(raw: str) -> tuple[Destination, ...]: + global _warned + if raw is None or isinstance(raw, str) and not raw.strip(): + return () + try: + if not isinstance(raw, str) or len(raw) > 32768: + raise ValueError("Configuration too large") + values = json.loads(raw) + if not isinstance(values, list) or len(values) > 8: + raise ValueError("Invalid exporters") + result = [] + for value in values: + if (not isinstance(value, dict) or set(value) != {"type", "endpoint", "allowed_origins", "credential_env"} + or value["type"] not in {"langfuse", "helicone"} + or not isinstance(value["credential_env"], str) + or not re.fullmatch(r"[A-Z][A-Z0-9_]{0,99}_EXPORT_CREDENTIAL", value["credential_env"]) + or not isinstance(value["allowed_origins"], list) or not 1 <= len(value["allowed_origins"]) <= 16): + raise ValueError("Invalid exporter") + origin = _origin(value["endpoint"]) + if origin not in {_origin(item, origin_only=True) for item in value["allowed_origins"]}: + raise ValueError("Destination not allowed") + item = Destination(value["type"], value["endpoint"], value["credential_env"]) + if item in result: + raise ValueError("Duplicate exporter") + result.append(item) + return tuple(result) + except (ValueError, TypeError, KeyError, RecursionError): + if not _warned: + _warned = True + logger.warning("Invalid OBSERVABILITY_EXPORTERS_JSON; exports disabled") + return () + + +def _text(value: Any) -> str | None: + return value if isinstance(value, str) and 0 < len(value) <= 256 and not re.search(r"[\x00-\x1f\x7f]", value) else None + + +def _number(value: Any) -> int | float | None: + return value if type(value) in {int, float} and 0 <= value <= 2**53 - 1 and math.isfinite(value) else None + + +def _tokens(value: Any) -> int | None: + return value if type(value) is int and 0 <= value <= 2**53 - 1 else None + + +def _timestamp(value: Any) -> str | None: + try: + if isinstance(value, str) and len(value) <= 40 and value.endswith("Z"): + moment = datetime.fromisoformat(value.replace("Z", "+00:00")) + elif type(value) is int and 0 <= value <= 253402300799000000000: + moment = datetime.fromtimestamp(value / 1_000_000_000, timezone.utc) + else: + return None + return moment.isoformat(timespec="milliseconds").replace("+00:00", "Z") + except (ValueError, OverflowError, OSError): + return None + + +def observation(record: dict, origin: str) -> dict | None: + """Allowlist scalar metadata; never traverse payloads, headers or error messages.""" + request_id = _text(record.get("request_id")) + principal = _text(record.get("principal")) + if not request_id or not principal or origin not in {"flask", "worker"}: + return None + status = _tokens(record.get("status")) + status = status if status is not None and 100 <= status <= 599 else None + basis = record.get("cost_basis") + basis = basis if isinstance(basis, str) and basis in _BASES else None + usage_basis = record.get("usage_basis") + usage_basis = usage_basis if isinstance(usage_basis, str) and usage_basis in _PROVENANCE else ( + "estimated" if basis == "estimate" else "provider" if basis == "usage" else "unknown") + provenance = record.get("provenance") + provenance = [item for item in provenance[:16] if isinstance(item, str) and item in _PROVENANCE] if isinstance(provenance, (list, tuple)) else [usage_basis] + kind = record.get("kind") + kind = kind if isinstance(kind, str) and kind in _KINDS else "proxy" + outcome = record.get("outcome") + outcome = outcome if isinstance(outcome, str) and outcome in _OUTCOMES else ( + "upstream_error" if status is not None and status >= 400 else "unknown") + end = _timestamp(record.get("end_ns")) or _timestamp(record.get("at")) + start = _timestamp(record.get("start_ns")) + latency = _number(record.get("latency_ms")) + if start is None and end is not None and latency is not None and latency <= 86_400_000: + start = (datetime.fromisoformat(end.replace("Z", "+00:00")) - timedelta(milliseconds=latency)).isoformat(timespec="milliseconds").replace("+00:00", "Z") + return {"request_id": request_id, "correlation_id": _text(record.get("trace_id")), + "principal": "principal:" + hashlib.sha256(principal.encode()).hexdigest(), "origin": origin, + "model": _text(record.get("selected_model")), "kind": kind, "status": status, + "error": outcome if outcome in {"upstream_error", "transport_error", "canceled"} else None, + "outcome": outcome, "start_time": start, "end_time": end, + "latency_ms": latency, "ttft_ms": _number(record.get("ttft_ms")), + "usage": {"input_tokens": _tokens(record.get("input_tokens")), "output_tokens": _tokens(record.get("output_tokens")), "basis": usage_basis}, + "cost": {"usd": _number(record.get("cost_usd")), "basis": basis}, "provenance": provenance} + + +def _event_id(item: dict) -> str: + return hashlib.sha256(json.dumps([item["origin"], item["request_id"], item["kind"]]).encode()).hexdigest() + + +def _helicone_time(value: str | None) -> dict | None: + if value is None: + return None + millis = int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000) + return {"seconds": millis // 1000, "milliseconds": millis % 1000} + + +def payloads(destination: Destination, batch: list[dict]) -> list[dict]: + if destination.type == "langfuse": + return [{"batch": [{"id": _event_id(item), "type": "generation-create", + "timestamp": item["end_time"], "body": { + "id": _event_id(item), "traceId": item["correlation_id"] or item["request_id"], + "name": f"multillm.{item['kind']}", "model": item["model"], + "startTime": item["start_time"], "endTime": item["end_time"], + "usage": {"input": item["usage"]["input_tokens"], "output": item["usage"]["output_tokens"], "unit": "TOKENS"}, + "metadata": item}} for item in batch]}] + return [{"providerRequest": {"url": "https://multillm.invalid", "json": {"model": item["model"]}, + "meta": {"Helicone-Request-Id": _event_id(item), "Helicone-User-Id": item["principal"], + "Helicone-Property-observation": json.dumps(item, separators=(",", ":"))}}, + "providerResponse": {"status": item["status"], "headers": {}, "json": { + "usage": {"prompt_tokens": item["usage"]["input_tokens"], "completion_tokens": item["usage"]["output_tokens"]}}}, + "timing": {"startTime": _helicone_time(item["start_time"]), "endTime": _helicone_time(item["end_time"])}} for item in batch] + + +def _receipt(response, payload: dict) -> int: + """A partial ingestion receipt acknowledges only explicit successful event IDs.""" + if response.status_code != 207: + return (len(payload["batch"]) if "batch" in payload else 1) if 200 <= response.status_code < 300 else 0 + if "batch" not in payload: + return 0 + try: + content = response.raw.read(32769) if hasattr(response, "raw") else response.content + if len(content) > 32768: + return 0 + receipt = json.loads(content) + if not isinstance(receipt, dict) or not isinstance(receipt.get("successes"), list) or not isinstance(receipt.get("errors"), list): + return 0 + successes = {item.get("id") for item in receipt["successes"] if isinstance(item, dict) and isinstance(item.get("id"), str)} + errors = {item.get("id") for item in receipt["errors"] if isinstance(item, dict) and isinstance(item.get("id"), str)} + return sum(item["id"] in successes and item["id"] not in errors for item in payload["batch"]) + except (AttributeError, ValueError, TypeError, RecursionError): + return 0 + + +class ObservabilityExporter: + def __init__(self, env: Mapping[str, str] | None = None, *, transport=None, auto_start: bool = True) -> None: + self.env = os.environ if env is None else env + self.transport = transport + self.auto_start = auto_start + self._raw: str | None = None + self._destinations: tuple[Destination, ...] = () + self._queue: deque = deque() + self._seen: dict[tuple, None] = {} + self._lock = threading.Lock() + self._delivery_lock = threading.Lock() + self._wake = threading.Event() + self._thread: threading.Thread | None = None + self._stats: dict[Destination, dict] = {} + self.accepted = self.dropped = 0 + + def _configure(self) -> tuple[Destination, ...]: + raw = self.env.get("OBSERVABILITY_EXPORTERS_JSON", "") + if raw != self._raw: + self._raw = raw + self._destinations = destinations(raw) + return self._destinations + + def submit(self, record: dict, *, origin: str = "flask") -> bool: + with self._lock: + targets = self._configure() + if not targets: + return False + item = observation(record, origin) + if item is None: + self.dropped += 1 + return False + identity = (origin, item["request_id"], item["kind"]) + if identity in self._seen: + return False + if len(self._queue) >= MAX_QUEUE: + self.dropped += 1 + return False + self._seen[identity] = None + if len(self._seen) > MAX_QUEUE: + del self._seen[next(iter(self._seen))] + self._queue.append((targets, item)) + self.accepted += 1 + if self.auto_start and (self._thread is None or not self._thread.is_alive()): + self._thread = threading.Thread(target=self._run, name="observability-export", daemon=True) + self._thread.start() + self._wake.set() + return True + + def _run(self) -> None: + while True: + self._wake.wait(5) + self._wake.clear() + try: + while self.export_once(): + pass + except Exception: + logger.warning("Observability export failed") + + def _deliver(self, target: Destination, batch: list[dict]) -> None: + stats = self._stats.setdefault(target, {"acknowledged": 0, "failed": 0, "attempts": 0, "delivery_status": "idle"}) + credential = self.env.get(target.credential_env, "") + if not isinstance(credential, str) or not credential or len(credential) > 8192 or re.search(r"[\x00-\x1f\x7f]", credential): + stats["failed"] += len(batch) + stats["delivery_status"] = "credential_unavailable" + return + authorization = "Basic " + base64.b64encode(credential.encode()).decode() if target.type == "langfuse" else "Bearer " + credential + for payload in payloads(target, batch): + count = len(batch) if target.type == "langfuse" else 1 + acknowledged = 0 + for _ in range(RETRIES + 1): + stats["attempts"] += 1 + try: + response = (self.transport or requests.post)(target.endpoint, json=payload, + headers={"Content-Type": "application/json", "Authorization": authorization}, + timeout=TIMEOUT, allow_redirects=False, stream=True) + try: + code = response.status_code + acknowledged = _receipt(response, payload) + if acknowledged or code != 429 and code < 500: + break + finally: + if hasattr(response, "close"): + response.close() + except Exception: + pass + stats["acknowledged"] += acknowledged + stats["failed"] += count - acknowledged + stats["delivery_status"] = "acknowledged" if acknowledged == count else "partial" if acknowledged else "failed" + + def export_once(self) -> int: + with self._delivery_lock: + with self._lock: + batch = [self._queue.popleft() for _ in range(min(BATCH_SIZE, len(self._queue)))] + groups: dict[Destination, list[dict]] = {} + for targets, item in batch: + for target in targets: + groups.setdefault(target, []).append(item) + for target, records in groups.items(): + self._deliver(target, records) + with self._lock: + retained = set(self._destinations) + for targets, _ in self._queue: + retained.update(targets) + self._stats = {target: stats for target, stats in self._stats.items() if target in retained} + return len(batch) + + def status(self) -> dict: + with self._lock: + return {"accepted": self.accepted, "dropped": self.dropped, "queued": len(self._queue), + "exporters": [{"type": target.type, "credential_env": target.credential_env, + **self._stats.get(target, {"acknowledged": 0, "failed": 0, "attempts": 0, "delivery_status": "idle"})} + for target in self._configure()]} + + +OBSERVABILITY_EXPORTER = ObservabilityExporter() diff --git a/services/telemetry_export.py b/services/telemetry_export.py index d43cd2b7..09433994 100644 --- a/services/telemetry_export.py +++ b/services/telemetry_export.py @@ -21,6 +21,8 @@ import requests +from services.observability_adapters import OBSERVABILITY_EXPORTER + logger = logging.getLogger(__name__) MAX_QUEUE = 2048 @@ -91,6 +93,10 @@ def __init__(self) -> None: self.stats = {"exported": 0, "dropped": 0, "failed_exports": 0} def submit(self, record: dict) -> bool: + try: + OBSERVABILITY_EXPORTER.submit(record, origin="flask") + except Exception: + logger.warning("Observability submission failed") if not configured(): return False with self._lock: diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 2c079f26..9a4d3140 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -303,6 +303,7 @@ "worker/media-signing.mjs", "worker/media-workflow.mjs", "worker/native-config-sync.mjs", + "worker/observability-export.mjs", "worker/opencode-session.mjs", "worker/opencode/reasoning-policy.mjs", "worker/opencode/reasoning-request.mjs", diff --git a/tests/test_observability_exports.py b/tests/test_observability_exports.py new file mode 100644 index 00000000..fd219ae7 --- /dev/null +++ b/tests/test_observability_exports.py @@ -0,0 +1,243 @@ +"""Content-free exports use fake collectors and independent delivery queues.""" +import json +import threading +from types import SimpleNamespace + +import pytest + +from services import observability_adapters as exports, telemetry_export + + +def settings(*types): + return {"OBSERVABILITY_EXPORTERS_JSON": json.dumps([ + {"type": kind, "endpoint": f"https://{kind}.example/ingest", + "allowed_origins": [f"https://{kind}.example"], "credential_env": f"TEST_{kind.upper()}_EXPORT_CREDENTIAL"} + for kind in types]), "TEST_LANGFUSE_EXPORT_CREDENTIAL": "public:synthetic-secret", "TEST_HELICONE_EXPORT_CREDENTIAL": "synthetic-secret"} + + +def record(**extra): + return {"request_id": "req-1", "trace_id": "a" * 32, "principal": "private-user", + "selected_model": "openai:test", "kind": "chat", "status": 200, + "start_ns": 1_000_000_000, "end_ns": 1_010_000_000, "latency_ms": 10, + "input_tokens": 0, "output_tokens": None, "cost_usd": None, "cost_basis": None, + "prompt": "private-prompt", "response": "private-output", "tools": ["private-tool"], + "headers": {"Authorization": "private-key"}, "key_prefix": "private-prefix", **extra} + + +def collector(calls, status=200): + def post(url, **kwargs): + calls.append((url, kwargs)) + return SimpleNamespace(status_code=status) + return post + + +@pytest.mark.parametrize("raw", [None, "", "[]"]) +def test_default_off_does_nothing(raw): + env = {} if raw is None else {"OBSERVABILITY_EXPORTERS_JSON": raw} + calls = [] + exporter = exports.ObservabilityExporter(env, transport=collector(calls), auto_start=False) + assert not exporter.submit(record()) + assert exporter.export_once() == 0 and calls == [] + assert exporter.status()["accepted"] == 0 + + +@pytest.mark.parametrize("raw", ["private-invalid", "{}", '[{"type":"unknown"}]', + '[{"type":"helicone","endpoint":"http://collector.example","allowed_origins":["http://collector.example"],"credential_env":"TEST_KEY"}]']) +def test_malformed_disables_with_one_content_free_warning(raw, caplog, monkeypatch): + monkeypatch.setattr(exports, "_warned", False) + exporter = exports.ObservabilityExporter({"OBSERVABILITY_EXPORTERS_JSON": raw}, auto_start=False) + assert not exporter.submit(record()) and not exporter.submit(record()) + assert len(caplog.records) == 1 and raw not in caplog.text + + +@pytest.mark.parametrize("change", [ + {"endpoint": "https://other.example/log"}, {"endpoint": "https://secret@helicone.example/log"}, + {"endpoint": "https://helicone.example/log?key=secret"}, {"endpoint": "https://helicone.example/log#secret"}, + {"credential_env": "secret-value!"}, {"credential_env": "GEMINI_API_KEY"}, {"api_key": "secret-value"}, + {"allowed_origins": ["https://helicone.example/path"]}, +]) +def test_destinations_and_credential_references_are_strict(change): + env = settings("helicone") + config = json.loads(env["OBSERVABILITY_EXPORTERS_JSON"]) + config[0].update(change) + env["OBSERVABILITY_EXPORTERS_JSON"] = json.dumps(config) + assert not exports.ObservabilityExporter(env, auto_start=False).submit(record()) + + +def test_adapter_schema_nulls_content_and_safe_status(): + calls = [] + exporter = exports.ObservabilityExporter(settings("langfuse", "helicone"), transport=collector(calls), auto_start=False) + assert exporter.submit(record()) and calls == [] + assert exporter.status()["accepted"] == 1 + assert exporter.status()["exporters"][0]["acknowledged"] == 0 + assert exporter.export_once() == 1 + langfuse = calls[0][1]["json"]["batch"][0] + assert langfuse["type"] == "generation-create" + assert langfuse["body"]["usage"] == {"input": 0, "output": None, "unit": "TOKENS"} + observation = langfuse["body"]["metadata"] + assert observation["cost"] == {"usd": None, "basis": None} + assert observation["model"] == "openai:test" and observation["latency_ms"] == 10 + assert observation["principal"].startswith("principal:") + helicone = calls[1][1]["json"] + assert helicone["providerResponse"]["json"]["usage"]["completion_tokens"] is None + assert helicone["providerRequest"]["json"] == {"model": "openai:test"} + assert helicone["timing"] == {"startTime": {"seconds": 1, "milliseconds": 0}, "endTime": {"seconds": 1, "milliseconds": 10}} + for _, kwargs in calls: + assert kwargs["timeout"] == 2 and kwargs["allow_redirects"] is False + text = json.dumps(kwargs["json"]) + for forbidden in ["private-user", "private-prompt", "private-output", "private-tool", "private-key", "private-prefix"]: + assert forbidden not in text + status = exporter.status() + assert "synthetic-secret" not in json.dumps(status) + assert all(item["acknowledged"] == 1 and item["delivery_status"] == "acknowledged" for item in status["exporters"]) + + +def test_queue_overflow_batch_bound_dedup_and_kind_cost_isolation(): + calls = [] + exporter = exports.ObservabilityExporter(settings("langfuse"), transport=collector(calls), auto_start=False) + for index in range(1000): + assert exporter.submit(record(request_id=f"req-{index}")) + assert not exporter.submit(record(request_id="overflow")) + assert exporter.status()["dropped"] == 1 and exporter.status()["queued"] == 1000 + assert not exporter.submit(record()) + assert exporter.export_once() == 50 and len(calls[0][1]["json"]["batch"]) == 50 + for kind, cost in [("shadow", .1), ("canary", .2)]: + assert exporter.submit(record(request_id="same", kind=kind, cost_usd=cost, cost_basis="usage")) + while exporter.export_once(): + pass + observations = [event["body"]["metadata"] for _, args in calls for event in args["json"]["batch"]] + assert [(item["kind"], item["cost"]["usd"]) for item in observations[-2:]] == [("shadow", .1), ("canary", .2)] + + +def test_bounded_retries_independent_failure_no_secret_logs(caplog): + calls = [] + def post(url, **kwargs): + calls.append((url, kwargs)) + if "langfuse" in url: + raise RuntimeError("synthetic-secret private-prompt") + return SimpleNamespace(status_code=200) + exporter = exports.ObservabilityExporter(settings("langfuse", "helicone"), transport=post, auto_start=False) + exporter.submit(record()) + assert exporter.export_once() == 1 + assert len(calls) == 4 + assert exporter.status()["exporters"][0]["failed"] == 1 + assert exporter.status()["exporters"][1]["acknowledged"] == 1 + assert "synthetic-secret" not in caplog.text and "private-prompt" not in caplog.text + assert len({args["json"]["batch"][0]["id"] for url, args in calls if "langfuse" in url}) == 1 + + +@pytest.mark.parametrize("status", [301, 400, 401, 207]) +def test_rejected_or_partial_receipts_are_not_acknowledged(status): + calls = [] + exporter = exports.ObservabilityExporter(settings("langfuse"), transport=collector(calls, status), auto_start=False) + exporter.submit(record()); exporter.export_once() + assert len(calls) == 1 and exporter.status()["exporters"][0]["failed"] == 1 + + +def test_missing_credentials_are_failures_without_transport(): + env = settings("helicone"); env["TEST_HELICONE_EXPORT_CREDENTIAL"] = "" + calls = [] + exporter = exports.ObservabilityExporter(env, transport=collector(calls), auto_start=False) + exporter.submit(record()); exporter.export_once() + assert calls == [] and exporter.status()["exporters"][0]["delivery_status"] == "credential_unavailable" + + +def test_background_delivery_never_waits_in_submit(): + entered, release = threading.Event(), threading.Event() + def slow(*args, **kwargs): + entered.set(); release.wait(5) + return SimpleNamespace(status_code=200) + exporter = exports.ObservabilityExporter(settings("helicone"), transport=slow) + try: + assert exporter.submit(record()) + assert entered.wait(2) + assert exporter.submit(record(request_id="req-2")) + assert exporter.status()["accepted"] == 2 + finally: + release.set() + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_otlp_payload_and_return_contract_are_unchanged(monkeypatch, enabled): + calls = [] + env = settings("helicone") if enabled else {} + external = exports.ObservabilityExporter(env, transport=collector([]), auto_start=False) + monkeypatch.setattr(telemetry_export, "OBSERVABILITY_EXPORTER", external) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otlp.example") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "") + monkeypatch.setenv("OTEL_SERVICE_NAME", "multillm-proxy") + monkeypatch.setattr(telemetry_export.requests, "post", collector(calls)) + monkeypatch.setattr(telemetry_export.secrets, "token_hex", lambda length: "b" * (length * 2)) + monkeypatch.setattr(telemetry_export.time, "time_ns", lambda: 2_000_000_000) + exporter = telemetry_export.TelemetryExporter() + monkeypatch.setattr(exporter, "_run", lambda: None) + event = record() + expected_span = telemetry_export.span(event) + expected_metrics = telemetry_export.metrics([event]) + assert exporter.submit(event) + assert exporter.export_once() == 1 + assert calls[0][1]["json"]["resourceSpans"][0]["scopeSpans"][0]["spans"] == [expected_span] + assert calls[1][1]["json"]["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] == expected_metrics + assert external.status()["accepted"] == int(enabled) + + +def test_flask_exports_without_otlp_and_once_per_finalization(monkeypatch): + external = exports.ObservabilityExporter(settings("helicone"), auto_start=False) + monkeypatch.setattr(telemetry_export, "OBSERVABILITY_EXPORTER", external) + for name in ["OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"]: + monkeypatch.setenv(name, "") + exporter = telemetry_export.TelemetryExporter() + assert not exporter.submit(record()) # Existing return value describes the OTLP queue. + exporter.submit(record()) + assert external.status()["accepted"] == 1 and not exporter._queue + + +def test_partial_langfuse_receipt_only_acknowledges_matching_successes(): + def post(url, **kwargs): + first, second = kwargs["json"]["batch"] + return SimpleNamespace(status_code=207, content=json.dumps({ + "successes": [{"id": first["id"]}, {"id": "foreign"}], "errors": [{"id": second["id"]}]}).encode()) + exporter = exports.ObservabilityExporter(settings("langfuse"), transport=post, auto_start=False) + exporter.submit(record()); exporter.submit(record(request_id="req-2")) + exporter.export_once() + status = exporter.status()["exporters"][0] + assert (status["acknowledged"], status["failed"], status["attempts"], status["delivery_status"]) == (1, 1, 1, "partial") + + +@pytest.mark.parametrize("value", [True, -1, float("nan"), float("inf"), "2", {}]) +def test_untrusted_numeric_and_nested_metadata_are_not_exported(value): + item = exports.observation(record(input_tokens=value, cost_usd=value, error={"message": "private"}, provenance=[{}, "private", "provider"]), "flask") + assert item["usage"]["input_tokens"] is None and item["cost"]["usd"] is None + assert item["error"] is None and item["provenance"] == ["provider"] + + +def test_finalized_accounting_response_exports_once_with_exact_bytes(monkeypatch): + from flask import Flask, Response, g + from services import request_accounting as accounting + external = exports.ObservabilityExporter(settings("helicone"), auto_start=False) + monkeypatch.setattr(accounting.telemetry_export, "OBSERVABILITY_EXPORTER", external) + monkeypatch.setattr(accounting.usage_ledger.LEDGER, "record", lambda row: True) + monkeypatch.setattr(accounting.BudgetService, "settle", lambda *args, **kwargs: None) + monkeypatch.setattr(accounting.BudgetService, "record_cost", lambda row: None) + monkeypatch.setenv("USAGE_RESERVATIONS_ENABLED", "false") + monkeypatch.setenv("PROMPT_CACHE_USAGE_BUCKETS_ENABLED", "false") + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", "{}") + for name in ["OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"]: + monkeypatch.setenv(name, "") + app = Flask(__name__) + raw = b'{"choices":[{"message":{"content":"private-output"}}]}' + @app.post("/v1/chat/completions") + def complete(): + context = accounting.UsageContext(kind="chat", models=["openai:test"], provider="openai", + user={"username": "private-user"}, started=accounting.time.perf_counter(), start_ns=1, + input_tokens=1, output_tokens=1, path="/v1/chat/completions", trace=("a" * 32, None), request_id="req-finalized") + g.usage_context = context + g.request_id = "req-finalized" + g.multillm_model = "openai:test" + response = accounting.finish(Response(raw, headers={"X-Request-ID": "unchanged"})) + return accounting.finish(response) + response = app.test_client().post("/v1/chat/completions") + assert response.data == raw and response.headers["X-Request-ID"] == "unchanged" + assert external.status()["accepted"] == 1 diff --git a/tests/test_observability_exports_worker.mjs b/tests/test_observability_exports_worker.mjs new file mode 100644 index 00000000..c9406558 --- /dev/null +++ b/tests/test_observability_exports_worker.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ObservabilityExporter, setObservabilityExporter, createNativeObservabilityHook } from "../worker/observability-export.mjs"; +import { handleUsageLedgerRequest, recordNativeUsage } from "../worker/usage-ledger-d1.mjs"; + +const settings = (...types) => ({ OBSERVABILITY_EXPORTERS_JSON: JSON.stringify(types.map(type => ({ + type, endpoint: `https://${type}.example/ingest`, allowed_origins: [`https://${type}.example`], credential_env: `TEST_${type.toUpperCase()}_EXPORT_CREDENTIAL`, +}))), TEST_LANGFUSE_EXPORT_CREDENTIAL: "public:synthetic-secret", TEST_HELICONE_EXPORT_CREDENTIAL: "synthetic-secret" }); +const record = (extra = {}) => ({ request_id: "req-1", trace_id: "a".repeat(32), principal: "private-user", + selected_model: "openai:test", kind: "chat", status: 200, at: "2026-10-09T00:00:00.000Z", latency_ms: 10, + input_tokens: 0, output_tokens: null, cost_usd: null, cost_basis: null, + prompt: "private-prompt", response: "private-output", tools: ["private-tool"], headers: { authorization: "private-key" }, key_prefix: "private-prefix", ...extra }); +const collector = (calls, status = 200) => async (url, options) => { calls.push({ url, options, body: JSON.parse(options.body) }); return new Response("{}", { status }); }; + +test("defaults and malformed configuration never call transport", async () => { + const logs = [], original = console.warn; console.warn = message => logs.push(message); + try { + for (const raw of [undefined, "", "[]", "private-invalid", "{}", '[{"type":"unknown"}]']) { + const calls = [], exporter = new ObservabilityExporter({ OBSERVABILITY_EXPORTERS_JSON: raw }, { transport: collector(calls) }); + assert.equal(exporter.submit(record()), false); assert.equal(exporter.submit(record()), false); + assert.equal(await exporter.flush(), 0); assert.equal(calls.length, 0); + } + assert.equal(logs.length, 1); assert.ok(!logs.join("").includes("private-invalid")); + } finally { console.warn = original; } +}); + +test("destination allowlists and credential names reject unsafe configuration", () => { + for (const change of [{ endpoint: "http://helicone.example/log" }, { endpoint: "https://other.example/log" }, + { endpoint: "https://secret@helicone.example/log" }, { endpoint: "https://helicone.example/log?key=secret" }, + { endpoint: "https://helicone.example/log#secret" }, { api_key: "secret" }, { credential_env: "secret!" }, { credential_env: "GEMINI_API_KEY" }, + { allowed_origins: ["https://helicone.example/path"] }]) { + const env = settings("helicone"), config = JSON.parse(env.OBSERVABILITY_EXPORTERS_JSON); + Object.assign(config[0], change); env.OBSERVABILITY_EXPORTERS_JSON = JSON.stringify(config); + assert.equal(new ObservabilityExporter(env).submit(record()), false); + } +}); + +test("adapter schemas preserve nulls, strip content, and expose only credential references", async () => { + const calls = [], exporter = new ObservabilityExporter(settings("langfuse", "helicone"), { transport: collector(calls) }); + assert.equal(exporter.submit(record()), true); assert.equal(calls.length, 0); + assert.equal(exporter.status().exporters[0].acknowledged, 0); + assert.equal(await exporter.flush(), 1); + const event = calls[0].body.batch[0]; assert.equal(event.type, "generation-create"); + assert.deepEqual(event.body.usage, { input: 0, output: null, unit: "TOKENS" }); + assert.deepEqual(event.body.metadata.cost, { usd: null, basis: null }); + assert.match(event.body.metadata.principal, /^principal:[0-9a-f]{64}$/); + assert.equal(calls[1].body.providerResponse.json.usage.completion_tokens, null); + assert.deepEqual(calls[1].body.providerRequest.json, { model: "openai:test" }); + assert.deepEqual(calls[1].body.timing, { startTime: { seconds: 1791503999, milliseconds: 990 }, endTime: { seconds: 1791504000, milliseconds: 0 } }); + for (const call of calls) { + assert.equal(call.options.redirect, "error"); assert.ok(call.options.signal); + for (const value of ["private-user", "private-prompt", "private-output", "private-tool", "private-key", "private-prefix"]) assert.ok(!JSON.stringify(call.body).includes(value)); + } + assert.ok(!JSON.stringify(exporter.status()).includes("synthetic-secret")); + assert.deepEqual(exporter.status().exporters.map(item => item.acknowledged), [1, 1]); +}); + +test("overflow, bounded batches, deduplication and separate experiment costs", async () => { + const calls = [], exporter = new ObservabilityExporter(settings("langfuse"), { transport: collector(calls) }); + for (let index = 0; index < 1000; index++) assert.equal(exporter.submit(record({ request_id: `req-${index}` })), true); + assert.equal(exporter.submit(record({ request_id: "overflow" })), false); + assert.equal(exporter.status().dropped, 1); assert.equal(exporter.status().queued, 1000); + assert.equal(exporter.submit(record({ request_id: "req-1" })), false); + assert.equal(await exporter.exportOnce(), 50); assert.equal(calls[0].body.batch.length, 50); + for (const [kind, cost] of [["shadow", .1], ["canary", .2]]) assert.equal(exporter.submit(record({ request_id: "same", kind, cost_usd: cost, cost_basis: "usage" })), true); + await exporter.flush(); + const observations = calls.flatMap(call => call.body.batch.map(event => event.body.metadata)); + assert.deepEqual(observations.slice(-2).map(item => [item.kind, item.cost.usd]), [["shadow", .1], ["canary", .2]]); +}); + +test("timeouts and retries are bounded and failures are isolated", async () => { + const calls = [], logs = [], original = console.warn; + console.warn = value => logs.push(value); + const exporter = new ObservabilityExporter(settings("langfuse", "helicone"), { + transport: async (url, options) => { calls.push({ url, options }); if (url.includes("langfuse")) throw new Error("private-secret"); return new Response("{}"); }, + }); + try { + exporter.submit(record()); await exporter.flush(); + assert.equal(calls.length, 4); assert.equal(exporter.status().exporters[0].failed, 1); + assert.equal(exporter.status().exporters[1].acknowledged, 1); assert.ok(!logs.join("").includes("private-secret")); + assert.equal(new Set(calls.slice(0, 3).map(call => JSON.parse(call.options.body).batch[0].id)).size, 1); + const hanging = new ObservabilityExporter(settings("helicone"), { transport: () => new Promise(() => {}) }); + hanging.submit(record()); const start = performance.now(); await hanging.flush(); + assert.ok(performance.now() - start >= 5900 && performance.now() - start < 8000); + assert.equal(hanging.status().exporters[0].attempts, 3); + } finally { console.warn = original; } +}); + +test("redirects, authentication failures and partial receipts are not retried or acknowledged", async () => { + for (const status of [301, 400, 401, 207]) { + const calls = [], exporter = new ObservabilityExporter(settings("langfuse"), { transport: collector(calls, status) }); + exporter.submit(record()); await exporter.flush(); + assert.equal(calls.length, 1); assert.equal(exporter.status().exporters[0].failed, 1); + } +}); + +test("native exports once without waiting; forwarded ledger flushes do not export", async () => { + const calls = [], pending = [], rows = []; + const env = { ...settings("helicone"), INTELLIGENCE_DB: { + prepare(sql) { return { bind(...values) { return { sql, values }; } }; }, + async batch(statements) { rows.push(...JSON.parse(statements[1].values[0])); return [{ meta: { changes: 1 } }]; }, + } }; + const exporter = new ObservabilityExporter(env, { transport: collector(calls) }); setObservabilityExporter(env, exporter); + const event = { provider: "openai", model: "test", principal: "edge:trusted", requestId: crypto.randomUUID(), + endpoint: "/v1/chat/completions", status: 200, duration_ms: 10, input_tokens: 0, output_tokens: null, cost_usd: null, cost_basis: null }; + await recordNativeUsage(env, event); await recordNativeUsage(env, event); + assert.equal(exporter.status().accepted, 1); assert.equal(calls.length, 0); + const forwarded = { ...rows[0], request_id: "flask-request" }; delete forwarded.bucket; + const response = await handleUsageLedgerRequest(new Request("http://intelligence.internal/v1/usage", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, operation: "record", batch: "a".repeat(32), rows: [forwarded] }), + }), env); + assert.deepEqual(await response.json(), { version: 1, recorded: 1, duplicate: false }); + assert.equal(exporter.status().accepted, 1); + await recordNativeUsage(env, { ...event, requestId: crypto.randomUUID() }, { waitUntil(promise) { pending.push(promise); } }); + await Promise.all(pending); assert.equal(calls.length, 2); +}); + +test("exports work without D1 and concurrent flushes deliver each queue item once", async () => { + const calls = [], env = settings("helicone"), exporter = new ObservabilityExporter(env, { transport: collector(calls) }); + setObservabilityExporter(env, exporter); + const original = console.error; console.error = () => {}; + try { + await recordNativeUsage(env, { provider: "openai", model: "test", principal: "edge:trusted", requestId: crypto.randomUUID(), + endpoint: "/v1/chat/completions", status: 200, duration_ms: 10, input_tokens: null, output_tokens: null, cost_usd: null, cost_basis: null }); + await Promise.all([exporter.flush(), exporter.flush()]); assert.equal(calls.length, 1); + } finally { console.error = original; } +}); + +test("partial Langfuse receipts acknowledge only matching successful event IDs", async () => { + const exporter = new ObservabilityExporter(settings("langfuse"), { transport: async (url, options) => { + const [first, second] = JSON.parse(options.body).batch; + return Response.json({ successes: [{ id: first.id }, { id: "foreign" }], errors: [{ id: second.id }] }, { status: 207 }); + } }); + exporter.submit(record()); exporter.submit(record({ request_id: "req-2" })); await exporter.flush(); + const stats = exporter.status().exporters[0]; + assert.deepEqual([stats.acknowledged, stats.failed, stats.attempts, stats.delivery_status], [1, 1, 1, "partial"]); +}); + +test("slow partial response bodies share the delivery timeout", async () => { + const exporter = new ObservabilityExporter(settings("langfuse"), { transport: async () => + new Response(new ReadableStream({ pull() {} }), { status: 207 }) }); + exporter.submit(record()); const start = performance.now(); await exporter.flush(); + assert.ok(performance.now() - start >= 5900 && performance.now() - start < 8000); + assert.equal(exporter.status().exporters[0].failed, 1); +}); + +test("static native finalizer exports independently of metrics or storage", async () => { + const calls = [], pending = [], env = settings("helicone"); + const exporter = new ObservabilityExporter(env, { transport: collector(calls) }); setObservabilityExporter(env, exporter); + const hook = createNativeObservabilityHook(env, { waitUntil(promise) { pending.push(promise); } }); + assert.equal(hook.enabled(env), true); + const event = { provider: "openai", model: "test", principal: "edge:trusted", requestId: "native-only", + kind: "shadow", endpoint: "/v1/chat/completions", status: 200, duration_ms: 10, cost_usd: .01, cost_basis: "usage", input_tokens: 1, output_tokens: 0 }; + assert.equal(hook.finalize(event), true); assert.equal(hook.finalize(event), false); + await Promise.all(pending); assert.equal(calls.length, 1); + const observed = JSON.parse(calls[0].body.providerRequest.meta["Helicone-Property-observation"]); + assert.equal(observed.kind, "shadow"); assert.equal(observed.cost.usd, .01); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..3179223f 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,9 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "OBSERVABILITY_EXPORTERS_JSON", + "LANGFUSE_EXPORT_CREDENTIAL", + "HELICONE_EXPORT_CREDENTIAL", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS", diff --git a/worker/observability-export.mjs b/worker/observability-export.mjs new file mode 100644 index 00000000..da9500e8 --- /dev/null +++ b/worker/observability-export.mjs @@ -0,0 +1,253 @@ +/** Content-free, best-effort collector delivery. No generation transport is retried. */ +export const MAX_QUEUE = 1000; +export const BATCH_SIZE = 50; +export const TIMEOUT_MS = 2000; +export const RETRIES = 2; +let warned = false; +const exporters = new WeakMap(); +const kinds = new Set(["chat", "responses", "embeddings", "images", "audio", "proxy", "shadow", "canary", "roleplay"]); +const outcomes = new Set(["success", "unknown", "upstream_error", "transport_error", "canceled"]); +const bases = new Set(["usage", "estimate", "cache"]); +const provenanceValues = new Set(["provider", "request_estimate", "cache", "measured", "estimated", "unknown"]); +const text = value => typeof value === "string" && value.length > 0 && value.length <= 256 && !/[\x00-\x1f\x7f]/.test(value) ? value : null; +const number = value => typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER ? value : null; +const tokens = value => Number.isSafeInteger(value) && value >= 0 ? value : null; +const hex = bytes => [...new Uint8Array(bytes)].map(value => value.toString(16).padStart(2, "0")).join(""); +const digest = async value => hex(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value))); + +function origin(value, originOnly = false) { + if (typeof value !== "string" || value.length > 2048 || /[\s\x00-\x1f\x7f\\]/.test(value) || value.includes("?") || value.includes("#")) throw new Error("Invalid destination"); + const url = new URL(value); + if (url.protocol !== "https:" || !url.hostname || url.username || url.password || originOnly && url.pathname !== "/") throw new Error("Invalid destination"); + return url.origin; +} + +export function destinations(raw) { + if (raw === undefined || raw === null || raw === "" || typeof raw === "string" && !raw.trim()) return []; + try { + if (typeof raw !== "string" || raw.length > 32768) throw new Error("Invalid configuration"); + const values = JSON.parse(raw), result = []; + if (!Array.isArray(values) || values.length > 8) throw new Error("Invalid exporters"); + for (const value of values) { + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).sort().join(",") !== "allowed_origins,credential_env,endpoint,type" + || !["langfuse", "helicone"].includes(value.type) || typeof value.credential_env !== "string" + || !/^[A-Z][A-Z0-9_]{0,99}_EXPORT_CREDENTIAL$/.test(value.credential_env) + || !Array.isArray(value.allowed_origins) || value.allowed_origins.length < 1 || value.allowed_origins.length > 16) throw new Error("Invalid exporter"); + const allowed = value.allowed_origins.map(item => origin(item, true)); + if (!allowed.includes(origin(value.endpoint))) throw new Error("Destination not allowed"); + const target = Object.freeze({ type: value.type, endpoint: value.endpoint, credential_env: value.credential_env }); + if (result.some(item => JSON.stringify(item) === JSON.stringify(target))) throw new Error("Duplicate exporter"); + result.push(target); + } + return result; + } catch { + if (!warned) { warned = true; console.warn("Invalid OBSERVABILITY_EXPORTERS_JSON; exports disabled"); } + return []; + } +} + +function timestamp(value) { + if (typeof value !== "string" || value.length > 40 || !value.endsWith("Z")) return null; + const time = Date.parse(value); + return Number.isFinite(time) ? new Date(time).toISOString() : null; +} + +function observation(record, owner) { + const requestId = text(record.request_id), principal = text(record.principal); + if (!requestId || !principal || !["worker", "flask"].includes(owner)) return null; + const status = tokens(record.status), validStatus = status >= 100 && status <= 599 ? status : null; + const basis = bases.has(record.cost_basis) ? record.cost_basis : null; + const usageBasis = provenanceValues.has(record.usage_basis) ? record.usage_basis + : basis === "estimate" ? "estimated" : basis === "usage" ? "provider" : "unknown"; + const outcome = outcomes.has(record.outcome) ? record.outcome : validStatus >= 400 ? "upstream_error" : "unknown"; + const end = timestamp(record.at), latency = number(record.latency_ms); + const start = end !== null && latency !== null && latency <= 86_400_000 ? new Date(Date.parse(end) - latency) : null; + return { request_id: requestId, correlation_id: text(record.trace_id), principal, origin: owner, + model: text(record.selected_model), kind: kinds.has(record.kind) ? record.kind : "proxy", status: validStatus, + error: ["upstream_error", "transport_error", "canceled"].includes(outcome) ? outcome : null, outcome, + start_time: start && Number.isFinite(start.getTime()) ? start.toISOString() : null, end_time: end, + latency_ms: latency, ttft_ms: number(record.ttft_ms), + usage: { input_tokens: tokens(record.input_tokens), output_tokens: tokens(record.output_tokens), basis: usageBasis }, + cost: { usd: number(record.cost_usd), basis }, + provenance: Array.isArray(record.provenance) ? record.provenance.slice(0, 16).filter(item => provenanceValues.has(item)) : [usageBasis] }; +} + +async function payloads(target, records) { + const batch = await Promise.all(records.map(async record => { + const item = { ...record, principal: `principal:${await digest(record.principal)}` }; + const id = await digest(JSON.stringify([item.origin, item.request_id, item.kind])); + return { item, id }; + })); + if (target.type === "langfuse") return [{ batch: batch.map(({ item, id }) => ({ id, type: "generation-create", timestamp: item.end_time, + body: { id, traceId: item.correlation_id ?? item.request_id, name: `multillm.${item.kind}`, model: item.model, + startTime: item.start_time, endTime: item.end_time, usage: { input: item.usage.input_tokens, output: item.usage.output_tokens, unit: "TOKENS" }, metadata: item }, + })) }]; + const timing = value => value === null ? null : { seconds: Math.floor(Date.parse(value) / 1000), milliseconds: Date.parse(value) % 1000 }; + return batch.map(({ item, id }) => ({ providerRequest: { url: "https://multillm.invalid", json: { model: item.model }, + meta: { "Helicone-Request-Id": id, "Helicone-User-Id": item.principal, "Helicone-Property-observation": JSON.stringify(item) } }, + providerResponse: { status: item.status, headers: {}, json: { usage: { prompt_tokens: item.usage.input_tokens, completion_tokens: item.usage.output_tokens } } }, + timing: { startTime: timing(item.start_time), endTime: timing(item.end_time) } })); +} + +async function receipt(response, payload, signal) { + const count = payload.batch?.length ?? 1; + if (signal.aborted) { void response.body?.cancel().catch(() => {}); return 0; } + if (response.status !== 207) { + void response.body?.cancel().catch(() => {}); + return response.status >= 200 && response.status < 300 ? count : 0; + } + if (!payload.batch || !response.body) return 0; + const reader = response.body.getReader(), chunks = []; + const abort = () => { void reader.cancel().catch(() => {}); }; + signal.addEventListener("abort", abort, { once: true }); + let length = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + length += value.length; + if (length > 32768) { void reader.cancel().catch(() => {}); return 0; } + chunks.push(value); + } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } + const result = JSON.parse(new TextDecoder().decode(bytes)); + if (!Array.isArray(result.successes) || !Array.isArray(result.errors)) return 0; + const successes = new Set(result.successes.map(item => item?.id)), errors = new Set(result.errors.map(item => item?.id)); + return payload.batch.filter(item => successes.has(item.id) && !errors.has(item.id)).length; + } catch { return 0; } + finally { signal.removeEventListener("abort", abort); reader.releaseLock(); } +} + +async function send(transport, target, body, headers) { + const controller = new AbortController(); + let timer; + try { + return await Promise.race([ + Promise.resolve().then(async () => { + const response = await transport(target.endpoint, { method: "POST", headers, body: JSON.stringify(body), redirect: "error", signal: controller.signal }); + return { code: response.status, acknowledged: await receipt(response, body, controller.signal) }; + }), + new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(new Error("Collector timeout")); }, TIMEOUT_MS); }), + ]); + } finally { clearTimeout(timer); } +} + +const emptyStats = () => ({ acknowledged: 0, failed: 0, attempts: 0, delivery_status: "idle" }); + +export class ObservabilityExporter { + constructor(env, options = {}) { + this.env = env; + this.transport = options.transport ?? ((...args) => fetch(...args)); + this.targets = destinations(env.OBSERVABILITY_EXPORTERS_JSON); + this.queue = []; + this.seen = new Set(); + this.accepted = 0; + this.dropped = 0; + this.stats = new Map(this.targets.map(target => [target, emptyStats()])); + this.delivery = null; + } + + submit(record, { owner = "worker" } = {}) { + if (!this.targets.length) return false; + const item = observation(record, owner); + if (!item) { this.dropped++; return false; } + const identity = JSON.stringify([owner, item.request_id, item.kind]); + if (this.seen.has(identity)) return false; + if (this.queue.length >= MAX_QUEUE) { this.dropped++; return false; } + this.seen.add(identity); + if (this.seen.size > MAX_QUEUE) this.seen.delete(this.seen.values().next().value); + this.queue.push(item); this.accepted++; + return true; + } + + async deliver(target, batch) { + const stats = this.stats.get(target), credential = this.env[target.credential_env]; + if (typeof credential !== "string" || !credential || credential.length > 8192 || /[\x00-\x1f\x7f]/.test(credential)) { + stats.failed += batch.length; stats.delivery_status = "credential_unavailable"; return; + } + const encoded = target.type === "langfuse" ? btoa(String.fromCharCode(...new TextEncoder().encode(credential))) : null; + const headers = { "Content-Type": "application/json", Authorization: target.type === "langfuse" ? `Basic ${encoded}` : `Bearer ${credential}` }; + for (const payload of await payloads(target, batch)) { + const count = target.type === "langfuse" ? batch.length : 1; + let acknowledged = 0; + for (let attempt = 0; attempt <= RETRIES; attempt++) { + stats.attempts++; + try { + const response = await send(this.transport, target, payload, headers), code = response.code; + acknowledged = response.acknowledged; + if (acknowledged || code !== 429 && code < 500) break; + } catch { /* Collector failures never affect serving or another destination. */ } + } + stats.acknowledged += acknowledged; stats.failed += count - acknowledged; + stats.delivery_status = acknowledged === count ? "acknowledged" : acknowledged ? "partial" : "failed"; + } + } + + exportOnce() { + if (this.delivery) return this.delivery.then(() => this.exportOnce()); + const batch = this.queue.splice(0, BATCH_SIZE); + if (!batch.length) return Promise.resolve(0); + this.delivery = (async () => { + for (const target of this.targets) { + try { await this.deliver(target, batch); } + catch { const stats = this.stats.get(target); stats.failed += batch.length; stats.delivery_status = "failed"; } + } + return batch.length; + })().finally(() => { this.delivery = null; }); + return this.delivery; + } + + async flush() { + // Work remains bounded even if traffic continues to refill the queue. + let drained = 0; + while (drained < MAX_QUEUE) { + const count = await this.exportOnce(); + if (!count) break; + drained += count; + } + return drained; + } + + status() { + return { accepted: this.accepted, dropped: this.dropped, queued: this.queue.length, + exporters: this.targets.map(target => ({ type: target.type, credential_env: target.credential_env, ...this.stats.get(target) })) }; + } +} + +export function getObservabilityExporter(env) { + if (!exporters.has(env)) exporters.set(env, new ObservabilityExporter(env)); + return exporters.get(env); +} + +export function observabilityEnabled(env) { return getObservabilityExporter(env).targets.length > 0; } + +/** Explicit transport injection for tests and static runtime collaborators. */ +export function setObservabilityExporter(env, exporter) { exporters.set(env, exporter); } + +export function submitNativeObservation(env, record, ctx) { + try { + const exporter = getObservabilityExporter(env); + const accepted = exporter.submit(record); + if ((accepted || exporter.queue.length) && ctx?.waitUntil) ctx.waitUntil(Promise.resolve().then(() => exporter.flush())); + return accepted; + } catch { console.warn("Observability submission failed"); return false; } +} + +/** Native finalizer interface; forwarded Container records must never call it. */ +export function submitNativeEvent(env, event, ctx) { + const model = event.model ? `${event.provider}:${event.model}` : null; + const endpoint = typeof event.endpoint === "string" ? event.endpoint : ""; + const kind = endpoint.endsWith("/responses") ? "responses" : endpoint.endsWith("/embeddings") ? "embeddings" + : endpoint.endsWith("/images/generations") ? "images" : "chat"; + return submitNativeObservation(env, { at: new Date().toISOString(), principal: event.principal, + kind: event.kind ?? kind, selected_model: model, status: event.status, latency_ms: event.duration_ms, + input_tokens: event.input_tokens, output_tokens: event.output_tokens, cost_usd: event.cost_usd, + cost_basis: event.cost_basis, request_id: event.requestId, trace_id: event.trace_id, + ttft_ms: event.ttft_ms, outcome: event.outcome, usage_basis: event.usage_basis, provenance: event.provenance }, ctx); +} + +export function createNativeObservabilityHook(env, ctx) { + return { enabled: observabilityEnabled, finalize: event => submitNativeEvent(env, event, ctx) }; +} diff --git a/worker/usage-ledger-d1.mjs b/worker/usage-ledger-d1.mjs index e8fb53dc..b80704ad 100644 --- a/worker/usage-ledger-d1.mjs +++ b/worker/usage-ledger-d1.mjs @@ -6,6 +6,7 @@ */ import { boundedBody } from "./control-users-d1.mjs"; import { logFailure } from "./log.mjs"; +import { submitNativeEvent } from "./observability-export.mjs"; import { BUCKET_FIELDS, bucketInsertStatement, recordUsageWithBuckets, usageBucketsEnabled, validBaseUsageRow as validRow } from "./usage-buckets-d1.mjs"; // Upper bounds in milliseconds of the usage_daily latency buckets; the last bucket is open. @@ -192,7 +193,7 @@ export async function handleUsageLedgerRequest(request, env) { } /** Serialize native telemetry into the existing strict ledger; richer fields need a migration. */ -export async function recordNativeUsage(env, event) { +export async function recordNativeUsage(env, event, ctx) { const model = event.model ? `${event.provider}:${event.model}` : null; const kind = event.endpoint.endsWith("/responses") ? "responses" : event.endpoint.endsWith("/embeddings") ? "embeddings" : event.endpoint.endsWith("/images/generations") ? "images" : "chat"; @@ -202,6 +203,7 @@ export async function recordNativeUsage(env, event) { output_tokens: event.output_tokens, cost_usd: event.cost_usd, cost_basis: event.cost_basis, request_id: event.requestId }; if (!validRow(row)) throw new Error("Invalid native usage metadata"); + submitNativeEvent(env, event, ctx); if (!env.INTELLIGENCE_DB) { console.error(JSON.stringify({ event: "native_usage_storage_unavailable" })); return; } try { return await record(env.INTELLIGENCE_DB, env, { version: 1, operation: "record",