From 3d427eef10c14aee88562638515cef05947490cf Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:27:06 +0530 Subject: [PATCH 1/9] feat(auth): add opt-in SAML federation through a trusted broker SAML_ENABLED (off by default) adds broker-backed dashboard login. The gateway never parses SAML XML: an operator-run broker returns a compact JWS that is verified against pinned RS256 or Ed25519 keys and bound to the issuer, audience, recipient, nonce and browser state. Each request nonce is claimed once, and only an operator-created subject link maps a verified subject to an existing account, with no just-in-time accounts or administrator grants. Native ACS and metadata return 503 saml_native_unavailable. Request state, subject links and audit rows live in D1 (migration 0035). --- docs/saml-federation.md | 64 +++ .../0035_saml_federation.sql | 36 ++ package.json | 2 +- routes/saml_federation.py | 115 +++++ services/saml_federation.py | 366 ++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 9 + tests/test_saml_federation.py | 408 ++++++++++++++++++ tests/test_saml_federation_worker.mjs | 143 ++++++ worker/container-env.mjs | 4 + worker/d1-schema.mjs | 2 +- worker/saml-identity.mjs | 167 +++++++ 11 files changed, 1314 insertions(+), 2 deletions(-) create mode 100644 docs/saml-federation.md create mode 100644 intelligence-migrations/0035_saml_federation.sql create mode 100644 routes/saml_federation.py create mode 100644 services/saml_federation.py create mode 100644 tests/test_saml_federation.py create mode 100644 tests/test_saml_federation_worker.mjs create mode 100644 worker/saml-identity.mjs diff --git a/docs/saml-federation.md b/docs/saml-federation.md new file mode 100644 index 00000000..f221e21e --- /dev/null +++ b/docs/saml-federation.md @@ -0,0 +1,64 @@ +# SAML federation through a trusted broker + +The gateway accepts a signed identity token from an operator-run SAML broker. The broker validates the IdP response, XML signatures, destination, audience, timing and SAML request correlation. The gateway does not download IdP metadata, parse XML or implement native SAML validation. `/auth/saml/acs` and `/auth/saml/metadata` return JSON HTTP 503 `saml_native_unavailable` when federation is enabled. They do not publish an ACS descriptor. + +Federation is off by default. `SAML_ENABLED=false` or an empty value makes every `/auth/saml/*` and subject-link endpoint return HTTP 404 before session, CSRF or storage work. Existing gateway requests keep their behavior. Malformed configuration disables federation and emits one warning without configuration values. + +## Configuration + +Use an HTTPS broker URL without credentials or a fragment. Its query may contain broker-specific settings, but cannot contain `nonce`, `state` or `recipient`. Configure a fixed HTTPS callback URL ending in `/auth/saml/callback`, without a query or fragment. It is never derived from caller headers or a redirect parameter. + +```text +SAML_ENABLED=true +SAML_BROKER_URL=https://broker.example/login +SAML_CALLBACK_URL=https://gateway.example/auth/saml/callback +``` + +`SAML_TRUST_CONFIG_JSON` defaults to `{}`. An enabled deployment must supply exactly these fields: + +```json +{ + "issuer": "https://broker.example/issuer", + "audience": "gateway.example", + "keys": [ + {"kid": "broker-2026", "alg": "EdDSA", "public_key_pem": ""} + ], + "max_age_seconds": 300 +} +``` + +Each key pins one unique `kid` and exact algorithm: `RS256` with an RSA public key of at least 2048 bits, or `EdDSA` with an Ed25519 public key. There may be at most 16 keys. No private key belongs in gateway configuration. Symmetric algorithms, unsigned tokens, mismatched key types and remotely supplied key URLs are rejected. Rotate keys by temporarily pinning both public keys, then remove the old key after all assertions issued under it expire. `max_age_seconds` must be an integer from 1 to 300. The callback URL may instead be supplied as the Flask application's `SAML_CALLBACK_URL` configuration. + +## Login and callback + +`GET /auth/saml/login` stores SHA-256 digests of a random nonce, random request state and recipient with a five-minute expiry, then redirects HTTP 302 only to the configured broker. It also binds the state digest to the browser's signed dashboard session. A new login replaces that browser's active state. Caller `next` parameters are ignored. + +The broker returns one compact JWS in `token` and the request state in `state`, using `GET /auth/saml/callback` or a URL-encoded POST. Use exactly one of each field. A token must include `iss`, `aud`, `sub`, integer `iat` and `exp`, `nonce`, `state` and `recipient`. `aud` is one exact string. `state` is the original request ID; `nonce` and recipient must match the original login. A token's lifetime cannot exceed `max_age_seconds`; a fixed five-second clock skew is allowed by signature verification. Durable requests still expire at five minutes without extending their claim lifetime. + +The callback must retain the original browser session cookie. A top-level GET is compatible with a SameSite=Lax dashboard cookie; a cross-site POST requires an operator-reviewed secure cookie policy or a same-site broker. State binding replaces CSRF form-token validation only on the broker callback. GET tokens can appear in browser history and intermediary access logs: configure the broker and gateway edge to suppress or redact callback query strings. The gateway adds `Cache-Control: no-store` and `Referrer-Policy: no-referrer` and never stores assertion contents. + +The nonce is claimed atomically once. A replay or expired durable request returns HTTP 400 `saml_assertion_replayed`. Invalid signature, issuer, audience, key, algorithm, recipient or browser binding returns HTTP 400 `saml_assertion_invalid`. Unknown or duplicate callback fields return HTTP 400 `saml_request_invalid`. + +After verification, an active `(issuer, subject)` link must name an existing, unrevoked, unexpired gateway account. No link returns HTTP 403 `saml_subject_unlinked`; the gateway never creates accounts from broker attributes. Assertion roles, administrator flags and organisation names grant nothing. The stored link supplies organisation, team and grants revision; the injected `IdentityAuthority` must return that exact `TenantContext`. Its `IdentityAssertion` uses digests for identifiers so arbitrary broker subjects fit the enterprise opaque-ID contract. Account names outside that contract use `account:` as principal ID, while session issuance still receives the actual account name. + +The service requires a named dashboard session issuer that accepts `(account, TenantContext)` and returns `True` only after issuing the existing dashboard session. The default returns HTTP 503 `saml_session_unavailable`: API-key authentication has no separate session-creation function. The issuer must revalidate account permissions and populate the same session fields as dashboard login, using stored grants. The service uses the stored-link authority by default; an organisation authority can be injected to revalidate current membership. An unavailable authority returns HTTP 503 `saml_identity_unavailable`, and a scope mismatch returns HTTP 403 `saml_identity_denied`. + +## Administrator subject links + +These routes require an administrator dashboard session. PUT and DELETE also require the existing dashboard CSRF token. + +- `GET /admin/saml/links?offset=0` lists up to 100 links, including inactive links; use `offset` to paginate. +- `GET /admin/saml/links/` reads one link. +- `PUT /admin/saml/links` creates or updates the stable subject link. +- `PUT /admin/saml/links/` edits the same issuer/subject binding. It cannot rebind that identifier to a different subject. +- `DELETE /admin/saml/links/` deactivates the link and retains its row and audit history. + +PUT accepts `issuer`, `subject`, `account`, and optional `org_id`, `team_id`, `grants_revision`. `issuer` must equal the configured issuer. A team requires an organisation. Tenant IDs use the enterprise opaque-ID format. Omitting scope fields selects the linked account's legacy scope. Account existence is checked by the service and again against durable dashboard accounts in the private handler. Link identifiers are stable 32-character hex digests. Only issuer and subject digests are persisted, so operators must retain their own subject-to-account mapping outside the gateway. + +## Storage, audit and operational bounds + +Apply `0035_saml_federation.sql` to D1 before enabling federation. It adds `saml_requests`, `saml_subject_links` and `saml_audit` and preserves existing accounts. The private handler is `handleSamlIdentityRequest` at `http://intelligence.internal/v1/managed-state/saml`. Its dispatcher and private client endpoint must be registered, and `register_saml_federation_routes(app, csrf)` must be mounted. The core login guard must include `SAML_PUBLIC_ENDPOINTS` in its public endpoint allowlist. The link routes retain their administrator guard. Until storage wiring is available, login fails closed. + +Every private operation validates the required tables and columns before changing state. Missing storage or a missing migration returns JSON HTTP 503 `saml_storage_unavailable` without admitting login. There is no local-memory identity fallback or automatic replay of uncertain writes. Login audit records contain an issuer digest, linked account if known, outcome and time. `authorized` means signature, link and authority checks passed before session issuance; a later session failure adds its failure outcome. Link changes also record the operator's digest. Neither audit nor nonce rows contain tokens, raw subjects, nonce or request state. + +Assertions are limited to 16 KiB; private RPC bodies to 8 KiB; trust configuration to 64 KiB; subjects to 1,024 characters. Expiry prevents reuse but does not delete request rows. No automatic audit or link pruning is performed: retention and database capacity are operator responsibilities. Retain inactive links and their audit history. D1 operations and external broker hosting may incur their normal charges; this feature makes no model-provider calls and defines no additional billing rate. Live IdP compatibility, deployment and native XML validation require separate operational verification. diff --git a/intelligence-migrations/0035_saml_federation.sql b/intelligence-migrations/0035_saml_federation.sql new file mode 100644 index 00000000..340b701a --- /dev/null +++ b/intelligence-migrations/0035_saml_federation.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS saml_requests ( + state_digest TEXT PRIMARY KEY, + nonce_digest TEXT NOT NULL, + recipient_digest TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + claimed_at INTEGER, + CHECK(expires_at > created_at AND expires_at <= created_at + 300) +); +CREATE INDEX IF NOT EXISTS idx_saml_requests_expiry ON saml_requests(expires_at); + +CREATE TABLE IF NOT EXISTS saml_subject_links ( + id TEXT PRIMARY KEY, + issuer_digest TEXT NOT NULL, + subject_digest TEXT NOT NULL, + account TEXT NOT NULL REFERENCES control_users(username), + org_id TEXT, + team_id TEXT, + grants_revision INTEGER NOT NULL DEFAULT 0 CHECK(grants_revision >= 0), + active INTEGER NOT NULL DEFAULT 1 CHECK(active IN (0, 1)), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + CHECK(team_id IS NULL OR org_id IS NOT NULL), + UNIQUE(issuer_digest, subject_digest) +); +CREATE INDEX IF NOT EXISTS idx_saml_subject_links_account ON saml_subject_links(account, active); + +CREATE TABLE IF NOT EXISTS saml_audit ( + id TEXT PRIMARY KEY, + issuer_digest TEXT NOT NULL, + account TEXT, + actor_digest TEXT, + outcome TEXT NOT NULL, + occurred_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_saml_audit_time ON saml_audit(occurred_at); diff --git a/package.json b/package.json index c66e4a8f..b0ea1b81 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_saml_federation_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/saml_federation.py b/routes/saml_federation.py new file mode 100644 index 00000000..ca4f5cd4 --- /dev/null +++ b/routes/saml_federation.py @@ -0,0 +1,115 @@ +"""Broker callback and administrator-managed subject links.""" +from copy import deepcopy +from functools import wraps + +from flask import jsonify, redirect, request, session + +from services.saml_federation import MAX_TOKEN_BYTES, SamlError, SamlFederation, load_config + +SAML_PUBLIC_ENDPOINTS = frozenset({"saml_login", "saml_callback", "saml_acs", "saml_metadata"}) + + +def admin_dashboard_user(): + from services.auth_service import AuthService + if not AuthService.is_authenticated(): + raise SamlError("authentication_required", 401) + user = AuthService.get_current_user() + if not user or not user.get("is_admin"): + raise SamlError("admin_required", 403) + return user + + +def register_saml_federation_routes(app, csrf): + service = app.extensions.get("saml_federation") + if service is None: + service = SamlFederation(load_config(callback_url=app.config.get("SAML_CALLBACK_URL"))) + app.extensions["saml_federation"] = service + + def guarded(view): + @wraps(view) + def wrapped(*args, **kwargs): + try: + return view(*args, **kwargs) + except SamlError as error: + return jsonify(error.envelope()), error.status + return wrapped + + def gate(): + if request.path.startswith("/auth/saml/") or request.path == "/admin/saml/links" or request.path.startswith("/admin/saml/links/"): + if not service.config.enabled: + return jsonify({"error": "not_found"}), 404 + if request.content_length is not None and request.content_length > MAX_TOKEN_BYTES + 2048: + return jsonify(SamlError("saml_request_invalid").envelope()), 400 + return None + + app.before_request_funcs.setdefault(None, []).insert(0, gate) + + @app.after_request + def private_saml_response(response): + if request.path.startswith("/auth/saml/") or request.path.startswith("/admin/saml/links"): + response.headers["Cache-Control"] = "no-store" + response.headers["Referrer-Policy"] = "no-referrer" + return response + + @app.get("/auth/saml/login") + @guarded + def saml_login(): + destination, state_digest = service.begin() + session["saml_state_digest"] = state_digest + return redirect(destination, code=302) + + @app.route("/auth/saml/callback", methods=["GET", "POST"]) + @csrf.exempt + @guarded + def saml_callback(): + values = request.args if request.method == "GET" else request.form + if set(values) != {"token", "state"} or any(len(values.getlist(key)) != 1 for key in values): + raise SamlError("saml_request_invalid") + browser_state_digest = session.get("saml_state_digest", "") + original_session = deepcopy(dict(session)) + try: + service.complete(values["token"], values["state"], browser_state_digest) + except SamlError: + session.clear() + session.update(original_session) + raise + session["saml_state_digest"] = browser_state_digest + return redirect("/", code=302) + + @app.route("/auth/saml/acs", methods=["GET", "POST"]) + @csrf.exempt + @guarded + def saml_acs(): + raise SamlError("saml_native_unavailable", 503) + + @app.get("/auth/saml/metadata") + @guarded + def saml_metadata(): + raise SamlError("saml_native_unavailable", 503) + + @app.route("/admin/saml/links", methods=["GET", "PUT"]) + @guarded + def saml_links(): + user = admin_dashboard_user() + if request.method == "GET": + offset = request.args.get("offset", "0") + if not offset.isascii() or not offset.isdigit() or len(offset) > 6: + raise SamlError("saml_link_invalid") + return jsonify(service.list_links(int(offset))) + csrf.protect() + return jsonify(service.put_link(request.get_json(silent=True), user["username"])) + + @app.route("/admin/saml/links/", methods=["GET", "PUT", "DELETE"]) + @guarded + def saml_link(identifier): + user = admin_dashboard_user() + if request.method == "GET": + result = service.get_link(identifier) + linked = result.get("link") + if linked is None: + raise SamlError("saml_link_not_found", 404) + return jsonify({"version": 1, "link": linked}) + csrf.protect() + if request.method == "PUT": + return jsonify(service.put_link(request.get_json(silent=True), user["username"], identifier)) + return jsonify(service.deactivate_link(identifier, user["username"])) diff --git a/services/saml_federation.py b/services/saml_federation.py new file mode 100644 index 00000000..3ad1065e --- /dev/null +++ b/services/saml_federation.py @@ -0,0 +1,366 @@ +"""Pinned broker JWS verification and durable, explicitly linked identity.""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import logging +import os +import re +import secrets +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import lru_cache +from typing import Callable, Mapping +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ed25519, padding, rsa + +from services.enterprise_contract import AuthorityDenied, AuthorityOperation, IdentityAssertion, IdentityAuthority, TenantContext + +logger = logging.getLogger(__name__) +MAX_AGE = 300 +CLOCK_SKEW = 5 +MAX_TOKEN_BYTES = 16384 +PRIVATE_ENDPOINT = "saml" +OPAQUE = re.compile(r"[A-Za-z0-9_:.\-]{1,128}\Z") +RANDOM = re.compile(r"[A-Za-z0-9_-]{43}\Z") +DIGEST = re.compile(r"[a-f0-9]{64}\Z") +LINK_ID = re.compile(r"[a-f0-9]{32}\Z") + + +class SamlError(Exception): + def __init__(self, code: str, status: int = 400): + super().__init__(code) + self.code, self.status = code, status + + def envelope(self) -> dict: + return {"error": {"code": self.code, "message": "SAML federation request could not be completed.", "retryable": False}} + + +def digest(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _text(value: object, maximum: int = 1024) -> bool: + if not isinstance(value, str) or not 0 < len(value) <= maximum or re.search(r"[\x00-\x1f\x7f]", value): + return False + try: + value.encode("utf-8") + return True + except UnicodeError: + return False + + +def _json(value: str) -> dict: + def unique(pairs): + result = {} + for key, item in pairs: + if key in result: + raise ValueError("Duplicate JSON key") + result[key] = item + return result + result = json.loads(value, object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError("Invalid number"))) + if not isinstance(result, dict): + raise ValueError("JSON object required") + return result + + +def _https(value: object, *, callback: bool = False) -> str: + if not isinstance(value, str) or not _text(value, 2048) or any(character.isspace() for character in value) or "\\" in value: + raise ValueError("Invalid HTTPS target") + url = urlsplit(value) + if (url.scheme != "https" or not url.hostname or url.username is not None or url.password is not None + or url.fragment or url.port == 0): + raise ValueError("Invalid HTTPS target") + if callback: + if url.path != "/auth/saml/callback" or url.query: + raise ValueError("Invalid callback target") + elif any(key in {"nonce", "state", "recipient"} for key, _ in parse_qsl(url.query, keep_blank_values=True)): + raise ValueError("Reserved broker query parameter") + return value + + +@lru_cache(maxsize=1) +def _warn_once() -> None: + logger.warning("Invalid SAML federation configuration; federation disabled") + + +@dataclass(frozen=True) +class TrustKey: + alg: str + public_key: object + + +@dataclass(frozen=True) +class SamlConfig: + enabled: bool = False + broker_url: str = "" + callback_url: str = "" + issuer: str = "" + audience: str = "" + keys: Mapping[str, TrustKey] | None = None + max_age_seconds: int = MAX_AGE + + +def load_config(env: Mapping[str, str] | None = None, *, callback_url: str | None = None) -> SamlConfig: + env = os.environ if env is None else env + flag = env.get("SAML_ENABLED", "") + flag = flag.strip().lower() if isinstance(flag, str) else "invalid" + if flag in {"", "0", "false", "no", "off"}: + return SamlConfig() + if flag not in {"1", "true", "yes", "on"}: + _warn_once() + return SamlConfig() + try: + broker = _https(env.get("SAML_BROKER_URL", "")) + callback = _https(callback_url if callback_url is not None else env.get("SAML_CALLBACK_URL", ""), callback=True) + raw = env.get("SAML_TRUST_CONFIG_JSON", "") or "{}" + if not isinstance(raw, str) or len(raw) > 65536: + raise ValueError("Invalid trust document") + trust = _json(raw) + if set(trust) != {"issuer", "audience", "keys", "max_age_seconds"}: + raise ValueError("Invalid trust fields") + age = trust["max_age_seconds"] + if not _text(trust["issuer"]) or not _text(trust["audience"]) or type(age) is not int or not 1 <= age <= MAX_AGE: + raise ValueError("Invalid trust bounds") + if not isinstance(trust["keys"], list) or not 1 <= len(trust["keys"]) <= 16: + raise ValueError("Pinned keys required") + keys = {} + for entry in trust["keys"]: + if not isinstance(entry, dict) or set(entry) != {"kid", "alg", "public_key_pem"}: + raise ValueError("Invalid key fields") + pem = entry["public_key_pem"] + if (not _text(entry["kid"], 128) or entry["kid"] in keys or not isinstance(pem, str) + or not 1 <= len(pem) <= 8192): + raise ValueError("Invalid pinned key") + key = serialization.load_pem_public_key(entry["public_key_pem"].encode("ascii")) + alg = entry["alg"] + if not ((alg == "RS256" and isinstance(key, rsa.RSAPublicKey) and key.key_size >= 2048) + or (alg == "EdDSA" and isinstance(key, ed25519.Ed25519PublicKey))): + raise ValueError("Invalid pinned algorithm") + keys[entry["kid"]] = TrustKey(alg, key) + return SamlConfig(True, broker, callback, trust["issuer"], trust["audience"], keys, age) + except (ValueError, TypeError, UnicodeError, KeyError, RecursionError, UnsupportedAlgorithm): + _warn_once() + return SamlConfig() + + +def _decode(segment: str) -> bytes: + if not re.fullmatch(r"[A-Za-z0-9_-]+", segment): + raise ValueError("Invalid compact JWS") + result = base64.urlsafe_b64decode(segment + "=" * (-len(segment) % 4)) + if base64.urlsafe_b64encode(result).rstrip(b"=").decode() != segment: + raise ValueError("Noncanonical compact JWS") + return result + + +def verify_assertion(token: str, config: SamlConfig, now: int) -> dict: + """Verify only configured asymmetric keys; assertion attributes grant nothing.""" + try: + if not config.enabled or not isinstance(token, str) or len(token) > MAX_TOKEN_BYTES: + raise ValueError("Invalid token") + head, payload, signature = token.split(".") + header = _json(_decode(head).decode("utf-8")) + if set(header) - {"kid", "alg", "typ"} or header.get("typ", "JWT") != "JWT": + raise ValueError("Unsupported JWS header") + key = (config.keys or {}).get(header.get("kid")) + if key is None or header.get("alg") != key.alg: + raise ValueError("Unpinned key or algorithm") + signed_bytes = (head + "." + payload).encode("ascii") + if key.alg == "RS256" and isinstance(key.public_key, rsa.RSAPublicKey): + key.public_key.verify(_decode(signature), signed_bytes, padding.PKCS1v15(), hashes.SHA256()) + elif key.alg == "EdDSA" and isinstance(key.public_key, ed25519.Ed25519PublicKey): + key.public_key.verify(_decode(signature), signed_bytes) + else: + raise ValueError("Unsupported key") + claims = _json(_decode(payload).decode("utf-8")) + iat, exp = claims.get("iat"), claims.get("exp") + if (type(iat) is not int or type(exp) is not int or iat < 0 or exp >= 9007199254740991 + or not 0 < exp - iat <= config.max_age_seconds or iat > now + CLOCK_SKEW + or exp <= now - CLOCK_SKEW or now - iat > config.max_age_seconds + CLOCK_SKEW): + raise ValueError("Invalid assertion time") + if (claims.get("iss") != config.issuer or claims.get("aud") != config.audience + or claims.get("recipient") != config.callback_url or not _text(claims.get("sub")) + or not isinstance(claims.get("nonce"), str) or not RANDOM.fullmatch(claims["nonce"]) + or not isinstance(claims.get("state"), str) or not RANDOM.fullmatch(claims["state"])): + raise ValueError("Invalid assertion binding") + if "nbf" in claims and (type(claims["nbf"]) is not int or claims["nbf"] > now + CLOCK_SKEW): + raise ValueError("Assertion not active") + return claims + except (ValueError, TypeError, UnicodeError, KeyError, InvalidSignature, RecursionError): + raise SamlError("saml_assertion_invalid") from None + + +class SamlStore: + """Fixed private RPC operations, without volatile identity fallback or retries.""" + def __init__(self, call: Callable | None = None): + self._call = call + + def call(self, operation: str, **values) -> dict: + try: + body = {"version": 1, "operation": operation, **values} + if self._call is None: + from services.intelligence_d1_store import request_private_intelligence + result = request_private_intelligence(body, endpoint=PRIVATE_ENDPOINT) + else: + result = self._call(body) + if not isinstance(result, dict) or type(result.get("version")) is not int or result.get("version") != 1 or "error" in result: + raise ValueError("Invalid storage reply") + return result + except Exception: + raise SamlError("saml_storage_unavailable", 503) from None + + def ready(self) -> None: + if self.call("ready").get("ready") is not True: + raise SamlError("saml_storage_unavailable", 503) + + +def lookup_gateway_account(account: str) -> dict | None: + from services.auth_service import AuthService + return AuthService.get_user_record(account) + + +def issue_dashboard_session(account: str, context: TenantContext) -> bool: + """An explicit session collaborator is required when no standalone issuer exists.""" + raise SamlError("saml_session_unavailable", 503) + + +def _active_account(record: object, account: str, now: int) -> bool: + if not isinstance(record, dict) or record.get("username") != account or record.get("revoked_at"): + return False + expires = record.get("expires_at") + if expires: + try: + instant = datetime.fromisoformat(expires.replace("Z", "+00:00")) + if instant.tzinfo is None: + instant = instant.replace(tzinfo=timezone.utc) + return instant.timestamp() > now + except (TypeError, ValueError, AttributeError): + return False + return True + + +def stored_link_authority(assertion: IdentityAssertion, operation: AuthorityOperation) -> TenantContext: + """The caller constructs the context only from the active stored subject link.""" + if assertion.verified is not True: + raise AuthorityDenied("identity_unverified") + return operation.context + + +class SamlFederation: + def __init__(self, config: SamlConfig, store: SamlStore | None = None, *, + session_issuer: Callable = issue_dashboard_session, account_lookup: Callable = lookup_gateway_account, + identity_authority: IdentityAuthority = stored_link_authority, clock: Callable = time.time): + self.config, self.store = config, store or SamlStore() + self.session_issuer, self.account_lookup = session_issuer, account_lookup + self.identity_authority, self.clock = identity_authority, clock + + def begin(self) -> tuple[str, str]: + self.store.ready() + now, nonce, state = int(self.clock()), secrets.token_urlsafe(32), secrets.token_urlsafe(32) + result = self.store.call("create_request", state_digest=digest(state), nonce_digest=digest(nonce), + recipient_digest=digest(self.config.callback_url), now=now, expires_at=now + MAX_AGE) + if result.get("created") is not True: + raise SamlError("saml_storage_unavailable", 503) + url = urlsplit(self.config.broker_url) + query = parse_qsl(url.query, keep_blank_values=True) + [("nonce", nonce), ("state", state), ("recipient", self.config.callback_url)] + return urlunsplit((url.scheme, url.netloc, url.path, urlencode(query), "")), digest(state) + + def _audit(self, outcome: str, account: str | None = None) -> None: + result = self.store.call("audit", issuer_digest=digest(self.config.issuer), account=account, outcome=outcome, now=int(self.clock())) + if result.get("recorded") is not True: + raise SamlError("saml_storage_unavailable", 503) + + def complete(self, token: str, state: str, browser_state_digest: str) -> None: + self.store.ready() + account = None + try: + now = int(self.clock()) + claims = verify_assertion(token, self.config, now) + if (not isinstance(state, str) or state != claims["state"] or not isinstance(browser_state_digest, str) + or not DIGEST.fullmatch(browser_state_digest) or not hmac.compare_digest(digest(state), browser_state_digest)): + raise SamlError("saml_assertion_invalid") + claimed = self.store.call("claim_request", state_digest=digest(state), nonce_digest=digest(claims["nonce"]), + recipient_digest=digest(self.config.callback_url), now=now) + if claimed.get("claimed") is not True: + raise SamlError("saml_assertion_replayed") + linked = self.store.call("lookup_link", issuer_digest=digest(claims["iss"]), subject_digest=digest(claims["sub"])).get("link") + if not isinstance(linked, dict) or linked.get("active") != 1: + raise SamlError("saml_subject_unlinked", 403) + account = linked["account"] + if not _active_account(self.account_lookup(account), account, now): + raise SamlError("saml_subject_unlinked", 403) + principal = account if OPAQUE.fullmatch(account) else "account:" + digest(account) + context = TenantContext(principal, linked.get("org_id"), linked.get("team_id"), linked["grants_revision"]) + assertion = IdentityAssertion(digest(claims["iss"]), digest(claims["sub"]), digest(claims["aud"]), + digest(state), digest(claims["nonce"]), claims["exp"], True) + operation = AuthorityOperation(context, linked["id"], context.grants_revision, digest(state)) + try: + resolved = self.identity_authority(assertion, operation) + if type(resolved) is not TenantContext or resolved != context: + raise AuthorityDenied("authority_scope_mismatch") + except AuthorityDenied: + raise SamlError("saml_identity_denied", 403) from None + # Record permission before session mutation so an audit outage cannot admit a login. + self._audit("authorized", account) + try: + issued = self.session_issuer(account, context) + except Exception: + raise SamlError("saml_session_unavailable", 503) from None + if issued is not True: + raise SamlError("saml_session_unavailable", 503) + except SamlError as error: + if error.code != "saml_storage_unavailable": + self._audit(error.code, account) + raise + except Exception: + self._audit("saml_identity_unavailable", account) + raise SamlError("saml_identity_unavailable", 503) from None + + def list_links(self, offset: int = 0) -> dict: + self.store.ready() + return self.store.call("list_links", offset=offset) + + def get_link(self, identifier: str) -> dict: + self.store.ready() + if not LINK_ID.fullmatch(identifier): + raise SamlError("saml_link_invalid") + return self.store.call("get_link", id=identifier) + + def put_link(self, payload: object, actor: str, identifier: str | None = None) -> dict: + self.store.ready() + allowed = {"issuer", "subject", "account", "org_id", "team_id", "grants_revision"} + if not isinstance(payload, dict) or set(payload) - allowed or not {"issuer", "subject", "account"} <= set(payload): + raise SamlError("saml_link_invalid") + if payload["issuer"] != self.config.issuer or not _text(payload["subject"]) or not _text(payload["account"], 128): + raise SamlError("saml_link_invalid") + account = payload["account"] + try: + record = self.account_lookup(account) + except Exception: + raise SamlError("saml_identity_unavailable", 503) from None + if not _active_account(record, account, int(self.clock())): + raise SamlError("saml_subject_unlinked", 403) + try: + principal = account if OPAQUE.fullmatch(account) else "account:" + digest(account) + context = TenantContext(principal, payload.get("org_id"), payload.get("team_id"), payload.get("grants_revision", 0)) + if identifier is not None and not LINK_ID.fullmatch(identifier): + raise ValueError("Invalid link identifier") + except (ValueError, TypeError): + raise SamlError("saml_link_invalid") from None + # A subject has one stable link ID; edits retain its audit history. + identifier = identifier or digest(self.config.issuer + "\0" + payload["subject"])[:32] + linked = {"id": identifier, "issuer_digest": digest(payload["issuer"]), "subject_digest": digest(payload["subject"]), + "account": account, "org_id": context.org_id, "team_id": context.team_id, "grants_revision": context.grants_revision} + return self.store.call("put_link", link=linked, actor=actor, now=int(self.clock())) + + def deactivate_link(self, identifier: str, actor: str) -> dict: + self.store.ready() + if not LINK_ID.fullmatch(identifier): + raise SamlError("saml_link_invalid") + return self.store.call("deactivate_link", id=identifier, actor=actor, now=int(self.clock())) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..89918aae 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -161,6 +161,14 @@ "routes/responses_state.py": [ "app.route('/v1/responses/', methods=['GET', 'DELETE'])" ], + "routes/saml_federation.py": [ + "app.get('/auth/saml/login')", + "app.get('/auth/saml/metadata')", + "app.route('/admin/saml/links', methods=['GET', 'PUT'])", + "app.route('/admin/saml/links/', methods=['GET', 'PUT', 'DELETE'])", + "app.route('/auth/saml/acs', methods=['GET', 'POST'])", + "app.route('/auth/saml/callback', methods=['GET', 'POST'])" + ], "routes/shadow_eval.py": [ "app.get('/admin/workbench/shadow/league')", "app.get('/admin/workbench/shadow/samples')", @@ -398,6 +406,7 @@ "worker/roleplay/turn-trace.mjs", "worker/roleplay/validation.mjs", "worker/route-health-d1.mjs", + "worker/saml-identity.mjs", "worker/scheduled-maintenance.mjs", "worker/secret-digest.mjs", "worker/secret-firewall.mjs", diff --git a/tests/test_saml_federation.py b/tests/test_saml_federation.py new file mode 100644 index 00000000..e2a9a856 --- /dev/null +++ b/tests/test_saml_federation.py @@ -0,0 +1,408 @@ +"""Broker assertions are verified before durable identity or session work.""" +import base64 +import importlib +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import parse_qs, urlsplit +from unittest.mock import Mock, patch + +import pytest +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ed25519, padding, rsa +from flask import Flask, Response, session +from flask_wtf.csrf import CSRFProtect, generate_csrf + +from services import saml_federation as saml +from services.enterprise_contract import TenantContext +from routes.saml_federation import SAML_PUBLIC_ENDPOINTS, register_saml_federation_routes + +NOW = 1_800_000_000 +CALLBACK = "https://gateway.example/auth/saml/callback" +ISSUER = "https://broker.example/issuer" +AUDIENCE = "gateway.example" + + +class Store: + def __init__(self): + self.requests, self.links, self.audit, self.calls = {}, {}, [], [] + self.lock = threading.Lock() + self.available = True + + def __call__(self, body): + with self.lock: + self.calls.append(body) + if not self.available: + raise RuntimeError("private storage detail") + op = body["operation"] + if op == "ready": + return {"version": 1, "ready": True} + if op == "create_request": + self.requests[body["state_digest"]] = {**body, "claimed_at": None} + return {"version": 1, "created": True} + if op == "claim_request": + row = self.requests.get(body["state_digest"]) + ok = row and row["nonce_digest"] == body["nonce_digest"] and row["recipient_digest"] == body["recipient_digest"] and row["expires_at"] > body["now"] and row["claimed_at"] is None + if ok: + row["claimed_at"] = body["now"] + return {"version": 1, "claimed": bool(ok)} + if op == "lookup_link": + link = next((row for row in self.links.values() if row["issuer_digest"] == body["issuer_digest"] and row["subject_digest"] == body["subject_digest"] and row["active"]), None) + return {"version": 1, "link": link} + if op == "list_links": + return {"version": 1, "links": list(self.links.values())} + if op == "get_link": + return {"version": 1, "link": self.links.get(body["id"])} + if op == "put_link": + self.links[body["link"]["id"]] = {**body["link"], "active": 1} + self.audit.append(body) + return {"version": 1, "link": self.links[body["link"]["id"]]} + if op == "deactivate_link": + link = self.links.get(body["id"]) + if link: + link["active"] = 0 + self.audit.append(body) + return {"version": 1, "deactivated": bool(link)} + if op == "audit": + self.audit.append(body) + return {"version": 1, "recorded": True} + raise AssertionError(op) + + +@pytest.fixture(params=["EdDSA", "RS256"]) +def signer(request): + key = ed25519.Ed25519PrivateKey.generate() if request.param == "EdDSA" else rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = key.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode() + return request.param, key, pem + + +def settings(signer, **changes): + alg, _, pem = signer + return {"SAML_ENABLED": "true", "SAML_BROKER_URL": "https://broker.example/login?realm=enterprise", "SAML_CALLBACK_URL": CALLBACK, + "SAML_TRUST_CONFIG_JSON": json.dumps({"issuer": ISSUER, "audience": AUDIENCE, "keys": [{"kid": "pinned", "alg": alg, "public_key_pem": pem}], "max_age_seconds": 300}), **changes} + + +def signed(signer, claims, **header_changes): + alg, key, _ = signer + encode = lambda value: base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).rstrip(b"=") + data = encode({"alg": alg, "kid": "pinned", "typ": "JWT", **header_changes}) + b"." + encode(claims) + signature = key.sign(data) if alg == "EdDSA" else key.sign(data, padding.PKCS1v15(), hashes.SHA256()) + return (data + b"." + base64.urlsafe_b64encode(signature).rstrip(b"=")).decode() + + +@pytest.fixture +def registered(monkeypatch, signer): + env = settings(signer) + for key in ("SAML_ENABLED", "SAML_BROKER_URL", "SAML_CALLBACK_URL", "SAML_TRUST_CONFIG_JSON"): + monkeypatch.setenv(key, env[key]) + app = Flask(__name__) + app.config.update(SECRET_KEY="synthetic-saml-session", TESTING=True, WTF_CSRF_ENABLED=False) + csrf = CSRFProtect(app) + store, issue = Store(), Mock(return_value=True) + accounts = {"alice": {"username": "alice", "is_admin": False, "scopes": ["chat"], "revoked_at": None, "expires_at": None}} + service = saml.SamlFederation(saml.load_config(env), saml.SamlStore(store), session_issuer=issue, account_lookup=accounts.get, clock=lambda: NOW) + app.extensions["saml_federation"] = service + register_saml_federation_routes(app, csrf) + route_globals = register_saml_federation_routes.__globals__ + monkeypatch.setitem(route_globals, "admin_dashboard_user", lambda: {"username": "operator", "is_admin": True}) + app.add_url_rule("/csrf-fixture", view_func=lambda: {"token": generate_csrf()}) + return app, app.test_client(), service, store, issue, accounts, route_globals + + +def start(client): + response = client.get("/auth/saml/login?next=https://attacker.example") + assert response.status_code == 302 + url = urlsplit(response.location) + assert (url.scheme, url.netloc, url.path) == ("https", "broker.example", "/login") + query = parse_qs(url.query) + assert "next" not in query + return {"iss": ISSUER, "aud": AUDIENCE, "sub": "external:alice", "iat": NOW, "exp": NOW + 300, + "nonce": query["nonce"][0], "state": query["state"][0], "recipient": CALLBACK, "is_admin": True, "org_id": "foreign"} + + +def link(service, **changes): + return service.put_link({"issuer": ISSUER, "subject": "external:alice", "account": "alice", "org_id": "org:one", "team_id": "team:one", "grants_revision": 2, **changes}, "operator") + + +def callback(client, signer, claims, method="GET", **headers): + values = {"token": signed(signer, claims, **headers), "state": claims["state"]} + return client.get("/auth/saml/callback", query_string=values) if method == "GET" else client.post("/auth/saml/callback", data=values) + + +@pytest.mark.parametrize("flag", ["", "false", "0", "malformed"]) +def test_disabled_routes_before_auth_csrf_and_storage(registered, flag): + app, client, service, store, _, _, _ = registered + service.config = saml.load_config({"SAML_ENABLED": flag}) + app.config["WTF_CSRF_ENABLED"] = True + for path in ("login", "callback", "acs", "metadata", "unknown"): + for method in ("GET", "POST"): + assert client.open("/auth/saml/" + path, method=method).status_code == 404 + assert client.put("/admin/saml/links", json={}).status_code == 404 + assert store.calls == [] + + +def test_existing_response_is_unchanged(monkeypatch): + monkeypatch.setenv("SAML_ENABLED", "") + def baseline(): + app = Flask(__name__) + app.add_url_rule("/raw", view_func=lambda: Response(b"unchanged bytes", headers={"X-Existing": "same"})) + return app + before = baseline().test_client().get("/raw") + app = baseline() + register_saml_federation_routes(app, CSRFProtect()) + after = app.test_client().get("/raw") + assert (before.data, list(before.headers)) == (after.data, list(after.headers)) + + +@pytest.mark.parametrize("change", [{"SAML_BROKER_URL": "http://broker.example"}, {"SAML_BROKER_URL": "https://user:pass@broker.example"}, {"SAML_BROKER_URL": "https://broker.example/#fragment"}, {"SAML_BROKER_URL": "https://broker.example?nonce=bad"}, {"SAML_CALLBACK_URL": ""}, {"SAML_TRUST_CONFIG_JSON": "{}"}, {"SAML_TRUST_CONFIG_JSON": "private-invalid"}]) +def test_config_invalid_is_off_and_logs_once_without_values(signer, change, caplog): + saml._warn_once.cache_clear() + for _ in range(2): + assert not saml.load_config(settings(signer, **change)).enabled + assert len(caplog.records) == 1 + assert "private-invalid" not in caplog.text and "user:pass" not in caplog.text + + +@pytest.mark.parametrize("method", ["GET", "POST"]) +def test_valid_assertion_uses_only_existing_account_and_bound_authority(registered, signer, method): + _, client, service, store, issue, _, _ = registered + link(service) + claims = start(client) + authority = Mock(side_effect=lambda assertion, operation: operation.context) + service.identity_authority = authority + response = callback(client, signer, claims, method) + assert response.status_code == 302 and response.location == "/" + issue.assert_called_once_with("alice", TenantContext("alice", "org:one", "team:one", 2)) + assertion, operation = authority.call_args.args + assert assertion.verified and operation.context.org_id == "org:one" + assert response.headers["Cache-Control"] == "no-store" + content = json.dumps(store.requests) + json.dumps(store.audit) + assert claims["nonce"] not in content and claims["state"] not in content and "external:alice" not in content + assert callback(client, signer, claims).json["error"]["code"] == "saml_assertion_replayed" + assert issue.call_count == 1 + + +@pytest.mark.parametrize("change", [{"iss": "other"}, {"aud": "other"}, {"aud": [AUDIENCE, "other"]}, {"nonce": "wrong"}, {"recipient": "https://attacker.example"}, {"exp": NOW - 10}, {"iat": NOW + 20}, {"exp": NOW + 301}, {"iat": True}, {"exp": 1.2}, {"sub": ""}]) +def test_invalid_claims_never_issue_session(registered, signer, change): + _, client, service, _, issue, _, _ = registered + link(service) + response = callback(client, signer, {**start(client), **change}) + assert response.status_code == 400 + issue.assert_not_called() + + +@pytest.mark.parametrize("headers", [{"kid": "unknown"}, {"alg": "none"}, {"alg": "HS256"}, {"alg": "RS512"}, {"crit": ["private"]}, {"jku": "https://attacker.example"}]) +def test_untrusted_headers_are_rejected(registered, signer, headers): + _, client, service, _, issue, _, _ = registered + link(service) + assert callback(client, signer, start(client), **headers).status_code == 400 + issue.assert_not_called() + + +def test_signature_and_browser_binding_are_required(registered, signer): + app, client, service, store, issue, _, _ = registered + link(service) + claims = start(client) + token = signed(signer, claims) + head, body, sig = token.split(".") + assert client.get("/auth/saml/callback", query_string={"token": head + "." + body + "." + ("A" if sig[0] != "A" else "B") + sig[1:], "state": claims["state"]}).status_code == 400 + assert app.test_client().get("/auth/saml/callback", query_string={"token": token, "state": claims["state"]}).status_code == 400 + assert all(row["claimed_at"] is None for row in store.requests.values()) + issue.assert_not_called() + + +def test_unlinked_inactive_and_foreign_authority_cannot_grant(registered, signer): + _, client, service, store, issue, _, _ = registered + assert callback(client, signer, start(client)).json["error"]["code"] == "saml_subject_unlinked" + saved = link(service) + service.deactivate_link(saved["link"]["id"], "operator") + assert callback(client, signer, start(client)).status_code == 403 + link(service) + service.identity_authority = lambda *_: TenantContext("alice", "foreign") + assert callback(client, signer, start(client)).status_code == 403 + issue.assert_not_called() + assert store.links + + +def test_missing_storage_and_session_fail_closed(registered, signer): + _, client, service, store, issue, _, _ = registered + link(service) + claims = start(client) + store.available = False + for response in (client.get("/auth/saml/login"), callback(client, signer, claims), client.get("/admin/saml/links")): + assert response.status_code == 503 and response.json["error"]["code"] == "saml_storage_unavailable" + assert b"private storage" not in response.data + issue.assert_not_called() + store.available = True + service.session_issuer = saml.issue_dashboard_session + response = callback(client, signer, claims) + assert response.status_code == 503 and response.json["error"]["code"] == "saml_session_unavailable" + + +def test_revoked_or_missing_account_has_no_session(registered, signer): + _, client, service, _, issue, accounts, _ = registered + link(service) + accounts["alice"]["revoked_at"] = "2026-01-01" + assert callback(client, signer, start(client)).status_code == 403 + accounts.clear() + assert callback(client, signer, start(client)).status_code == 403 + issue.assert_not_called() + + +def test_admin_links_require_session_and_csrf_and_keep_history(registered): + app, client, _, store, _, _, route_module = registered + app.config["WTF_CSRF_ENABLED"] = True + payload = {"issuer": ISSUER, "subject": "external:alice", "account": "alice"} + assert client.put("/admin/saml/links", json=payload).status_code == 400 + token = client.get("/csrf-fixture").json["token"] + response = client.put("/admin/saml/links", json=payload, headers={"X-CSRFToken": token}) + assert response.status_code == 200 + identifier = response.json["link"]["id"] + assert client.delete("/admin/saml/links/" + identifier, headers={"X-CSRFToken": token}).json["deactivated"] + assert store.links[identifier]["active"] == 0 + route_module["admin_dashboard_user"] = Mock(side_effect=saml.SamlError("admin_required", 403)) + assert client.get("/admin/saml/links").status_code == 403 + assert len(store.audit) == 2 + + +def test_native_endpoints_never_claim_readiness(registered): + _, client, _, _, issue, _, _ = registered + for path in ("acs", "metadata"): + response = client.get("/auth/saml/" + path) + assert response.status_code == 503 and response.json["error"]["code"] == "saml_native_unavailable" + issue.assert_not_called() + + +def test_atomic_replay_claim(registered, signer): + _, client, service, _, issue, _, _ = registered + link(service) + claims = start(client) + def consume(_): + try: + service.complete(signed(signer, claims), claims["state"], saml.digest(claims["state"])) + return 200 + except saml.SamlError as error: + return error.status + with ThreadPoolExecutor(max_workers=2) as pool: + assert sorted(pool.map(consume, range(2))) == [200, 400] + assert issue.call_count == 1 + + +def test_session_issuer_failure_restores_browser_session(registered, signer): + _, client, service, _, _, _, _ = registered + link(service) + claims = start(client) + def partial_issue(*_): + session.clear() + session["authenticated"] = True + raise RuntimeError("private session detail") + service.session_issuer = partial_issue + response = callback(client, signer, claims) + assert response.status_code == 503 and response.json["error"]["code"] == "saml_session_unavailable" + with client.session_transaction() as saved: + assert "authenticated" not in saved and saved["saml_state_digest"] == saml.digest(claims["state"]) + assert b"private session detail" not in response.data + + +def test_audit_failure_prevents_session_issue(registered, signer): + _, client, service, store, issue, _, _ = registered + link(service) + original_call = store.__call__ + def failed_audit(body): + if body["operation"] == "audit": + raise RuntimeError("audit unavailable") + return original_call(body) + service.store = saml.SamlStore(failed_audit) + assert callback(client, signer, start(client)).json["error"]["code"] == "saml_storage_unavailable" + issue.assert_not_called() + + +def test_expired_request_cannot_use_fresh_token(registered, signer): + _, client, service, _, issue, _, _ = registered + link(service) + claims = start(client) + service.clock = lambda: NOW + 301 + claims.update(iat=NOW + 301, exp=NOW + 601) + response = callback(client, signer, claims) + assert response.json["error"]["code"] == "saml_assertion_replayed" + issue.assert_not_called() + + +def test_pinned_algorithm_mismatch_and_strict_trust_bounds(signer): + config = settings(signer) + trust = json.loads(config["SAML_TRUST_CONFIG_JSON"]) + for age in (0, 301, True, "300"): + assert not saml.load_config({**config, "SAML_TRUST_CONFIG_JSON": json.dumps({**trust, "max_age_seconds": age})}).enabled + for change in ({"alg": "HS256"}, {"alg": "none"}, {"alg": "RS256" if signer[0] == "EdDSA" else "EdDSA"}): + assert not saml.load_config({**config, "SAML_TRUST_CONFIG_JSON": json.dumps({**trust, "keys": [{**trust["keys"][0], **change}]})}).enabled + assert not saml.load_config({**config, "SAML_TRUST_CONFIG_JSON": json.dumps({**trust, "keys": trust["keys"] * 2})}).enabled + + +def test_request_binding_and_envelopes_cannot_be_spoofed(registered, signer): + _, client, service, _, issue, _, _ = registered + link(service) + claims = start(client) + token = signed(signer, claims) + assert client.get("/auth/saml/callback", query_string={"token": token, "state": "spoofed"}).status_code == 400 + assert client.post("/auth/saml/callback", json={"token": token, "state": claims["state"]}).status_code == 400 + assert client.get("/auth/saml/callback", query_string=[("token", token), ("token", token), ("state", claims["state"])]).status_code == 400 + assert client.get("/auth/saml/callback", query_string={"token": "x" * 17000, "state": claims["state"]}).status_code == 400 + issue.assert_not_called() + + +def test_admin_guard_requires_real_dashboard_administrator(monkeypatch): + routes = importlib.import_module(register_saml_federation_routes.__module__) + auth = importlib.import_module("services.auth_service").AuthService + monkeypatch.setattr(auth, "is_authenticated", lambda: False) + with pytest.raises(saml.SamlError) as failed: + routes.admin_dashboard_user() + assert failed.value.status == 401 + monkeypatch.setattr(auth, "is_authenticated", lambda: True) + monkeypatch.setattr(auth, "get_current_user", lambda: {"username": "alice", "is_admin": False}) + with pytest.raises(saml.SamlError) as failed: + routes.admin_dashboard_user() + assert failed.value.status == 403 + + +def test_application_callback_with_public_endpoint_registration(monkeypatch, tmp_path, signer): + env = {**settings(signer), "FLASK_SECRET_KEY": "synthetic-saml-session", "JWT_SECRET": "synthetic-saml-jwt", + "ADMIN_API_KEY": "synthetic-saml-admin", "AUTH_DB_PATH": str(tmp_path / "auth.sqlite3"), + "MODEL_REGISTRY_DB_PATH": str(tmp_path / "models.sqlite3"), "RATE_LIMIT_DB_PATH": str(tmp_path / "limits.sqlite3"), + "INTELLIGENCE_STORAGE_BACKEND": "", "CF_ACCESS_TEAM_DOMAIN": "", "CF_ACCESS_SSO_ONLY": "false"} + for key, value in env.items(): + monkeypatch.setenv(key, value) + with patch("config.load_runtime_env"), patch("env_loader.load_runtime_env"), patch("services.usage_ledger.start"), \ + patch("requests.Session.send", side_effect=AssertionError("No network")): + module = importlib.import_module("app") + with patch.object(module, "load_runtime_env"): + app = module.create_app() + app.config.update(TESTING=True, WTF_CSRF_ENABLED=True) + accounts = {"alice": {"username": "alice", "is_admin": False, "scopes": ["chat"], "api_key_prefix": "synthetic"}} + store = Store() + def issue(account, context): + assert context == TenantContext("alice") + session.clear() + session["authenticated"] = True + session["user"] = {"username": account, "is_admin": accounts[account]["is_admin"], "scopes": accounts[account]["scopes"], + "api_key_prefix": "synthetic", "session_id": "synthetic-session"} + return True + service = saml.SamlFederation(saml.load_config(env), saml.SamlStore(store), session_issuer=issue, account_lookup=accounts.get, clock=lambda: NOW) + app.extensions["saml_federation"] = service + register_saml_federation_routes(app, app.extensions["csrf"]) + link(service, org_id=None, team_id=None, grants_revision=0) + client = app.test_client() + # Exercise the existing guard first, then the required static allowlist integration. + blocked = client.get("/auth/saml/login") + assert blocked.status_code == 302 and urlsplit(blocked.location).path == "/login" + guard = next(hook for hook in app.before_request_funcs[None] if hook.__name__ == "handle_redirects") + monkeypatch.setitem(guard.__globals__, "PRODUCT_PUBLIC_ENDPOINTS", guard.__globals__["PRODUCT_PUBLIC_ENDPOINTS"] | SAML_PUBLIC_ENDPOINTS) + headers = next(hook for hook in app.after_request_funcs[None] if hook.__name__ == "add_response_headers") + for auth in {guard.__globals__["AuthService"], headers.__globals__["AuthService"]}: + monkeypatch.setattr(auth, "_load_user_by_username", accounts.get) + claims = start(client) + response = callback(client, signer, claims, "POST") + assert response.status_code == 302 + with client.session_transaction() as saved: + assert saved["authenticated"] is True and saved["user"]["is_admin"] is False + assert saved["user"]["scopes"] == ["chat"] + assert callback(client, signer, claims).json["error"]["code"] == "saml_assertion_replayed" diff --git a/tests/test_saml_federation_worker.mjs b/tests/test_saml_federation_worker.mjs new file mode 100644 index 00000000..e8db292f --- /dev/null +++ b/tests/test_saml_federation_worker.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; +import { handleSamlIdentityRequest } from "../worker/saml-identity.mjs"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; + +const hash = character => character.repeat(64); +const now = Math.floor(Date.now() / 1000); +const request = (body, url = "http://intelligence.internal/v1/managed-state/saml", headers = {}) => new Request(url, { + method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify({ version: 1, ...body }) }); + +function database(sql) { + return { prepare(query) { + let values = []; + return { bind(...args) { values = args; return this; }, first() { return sql.prepare(query).get(...values) ?? null; }, + all() { const stmt = sql.prepare(query); return stmt.columns().length ? { results: stmt.all(...values), meta: { changes: 0 } } + : { results: [], meta: { changes: Number(stmt.run(...values).changes) } }; }, run() { return this.all(); } }; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { const result = statements.map(statement => statement.all()); sql.exec("COMMIT"); return result; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; +} + +async function fixture(t) { + const sql = new DatabaseSync(":memory:"); t.after(() => sql.close()); + sql.exec(await readFile(new URL("../intelligence-migrations/0003_control_users.sql", import.meta.url), "utf8")); + sql.prepare("INSERT INTO control_users(username,api_key_hash,api_key_prefix,scopes,created_at) VALUES(?,?,?,?,?)").run("alice", "synthetic", "synthetic", "chat", "2026-01-01"); + const migration = await readFile(new URL("../intelligence-migrations/0035_saml_federation.sql", import.meta.url), "utf8"); + sql.exec(migration); sql.exec(migration); + const env = { SAML_ENABLED: "true", INTELLIGENCE_DB: database(sql) }; + return { sql, env, call: (body, settings = env) => handleSamlIdentityRequest(request(body), settings) }; +} + +test("default-off private handler reads no body or storage", async () => { + for (const flag of [undefined, "", "false", "0"]) { + const response = await handleSamlIdentityRequest(request({ operation: "ready" }), { SAML_ENABLED: flag, + INTELLIGENCE_DB: { prepare() { assert.fail("disabled storage"); } } }); + assert.equal(response.status, 404); + } +}); + +test("nonce digests expire and concurrent claim is exactly once", async t => { + const { sql, call } = await fixture(t); + const body = { state_digest: hash("a"), nonce_digest: hash("b"), recipient_digest: hash("c"), now, expires_at: now + 300 }; + assert.equal((await call({ operation: "create_request", ...body })).status, 200); + const claim = { operation: "claim_request", state_digest: body.state_digest, nonce_digest: body.nonce_digest, recipient_digest: body.recipient_digest, now }; + assert.equal((await (await call({ ...claim, nonce_digest: hash("d") })).json()).claimed, false); + const responses = await Promise.all([call(claim), call(claim)]); + const claimed = await Promise.all(responses.map(async response => (await response.json()).claimed)); + assert.deepEqual(claimed.sort(), [false, true]); + assert.equal(sql.prepare("SELECT claimed_at FROM saml_requests").get().claimed_at, now); + assert.equal((await (await call(claim)).json()).claimed, false); + assert.equal((await call({ operation: "create_request", ...body, state_digest: hash("e"), expires_at: now + 301 })).status, 400); + await call({ operation: "create_request", ...body, state_digest: hash("f"), expires_at: now + 1 }); + assert.equal((await (await call({ ...claim, state_digest: hash("f"), now: now + 1 })).json()).claimed, false); +}); + +test("links refer to existing accounts, remain scoped and retain audit and tombstones", async t => { + const { sql, call } = await fixture(t); + const link = { id: "a".repeat(32), issuer_digest: hash("a"), subject_digest: hash("b"), account: "alice", org_id: "org:one", team_id: "team:one", grants_revision: 2 }; + const put = { operation: "put_link", link, actor: "operator", now }; + assert.equal((await call({ ...put, link: { ...link, account: "missing" } })).status, 403); + assert.equal((await call(put)).status, 200); + assert.equal((await (await call({ operation: "lookup_link", issuer_digest: hash("c"), subject_digest: hash("b") })).json()).link, null); + assert.equal((await (await call({ operation: "lookup_link", issuer_digest: hash("a"), subject_digest: hash("b") })).json()).link.team_id, "team:one"); + assert.equal((await call({ operation: "deactivate_link", id: link.id, actor: "operator", now })).status, 200); + assert.equal(sql.prepare("SELECT active FROM saml_subject_links").get().active, 0); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM saml_audit").get().n, 2); + assert.match(sql.prepare("SELECT actor_digest FROM saml_audit LIMIT 1").get().actor_digest, /^[a-f0-9]{64}$/); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM control_users").get().n, 1); + assert.equal((await (await call({ operation: "list_links", offset: 0 })).json()).links.length, 1); +}); + +test("missing any table fails before writes and hides storage details", async t => { + const { call } = await fixture(t); + const sql = new DatabaseSync(":memory:"); t.after(() => sql.close()); + const response = await call({ operation: "ready" }, { SAML_ENABLED: "true", INTELLIGENCE_DB: database(sql) }); + assert.equal(response.status, 503); assert.equal((await response.json()).error.code, "saml_storage_unavailable"); +}); + +for (const table of ["saml_requests", "saml_subject_links", "saml_audit"]) test("missing " + table + " refuses request creation before mutations", async t => { + const { sql, call } = await fixture(t); + sql.exec(`DROP TABLE ${table}`); + const response = await call({ operation: "create_request", state_digest: hash("a"), nonce_digest: hash("b"), recipient_digest: hash("c"), now, expires_at: now + 300 }); + assert.equal(response.status, 503); assert.equal((await response.json()).error.code, "saml_storage_unavailable"); + if (table !== "saml_requests") assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM saml_requests").get().n, 0); +}); + +test("missing audit column fails before request creation", async t => { + const { sql, call } = await fixture(t); + sql.exec("ALTER TABLE saml_audit DROP COLUMN actor_digest"); + const response = await call({ operation: "create_request", state_digest: hash("a"), nonce_digest: hash("b"), recipient_digest: hash("c"), now, expires_at: now + 300 }); + assert.equal(response.status, 503); assert.equal((await response.json()).error.code, "saml_storage_unavailable"); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM saml_requests").get().n, 0); +}); + +test("audit transaction failure rolls back a subject link and existing accounts survive migration", async t => { + const { sql, call } = await fixture(t); + sql.exec("CREATE TRIGGER refuse_audit BEFORE INSERT ON saml_audit BEGIN SELECT RAISE(ABORT, 'synthetic audit outage'); END"); + const link = { id: "a".repeat(32), issuer_digest: hash("a"), subject_digest: hash("b"), account: "alice", org_id: null, team_id: null, grants_revision: 0 }; + assert.equal((await call({ operation: "put_link", link, actor: "operator", now })).status, 503); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM saml_subject_links").get().n, 0); + assert.equal(sql.prepare("SELECT username,scopes,is_admin FROM control_users").get().username, "alice"); + assert.equal(sql.prepare("SELECT scopes,is_admin FROM control_users").get().scopes, "chat"); + assert.equal(sql.prepare("SELECT is_admin FROM control_users").get().is_admin, 0); +}); + +test("changing a link binding cannot move its history or grant administrator attributes", async t => { + const { sql, call } = await fixture(t); + const link = { id: "a".repeat(32), issuer_digest: hash("a"), subject_digest: hash("b"), account: "alice", org_id: "org:one", team_id: null, grants_revision: 0 }; + assert.equal((await call({ operation: "put_link", link, actor: "operator", now })).status, 200); + assert.equal((await call({ operation: "put_link", link: { ...link, subject_digest: hash("c") }, actor: "operator", now })).status, 409); + assert.equal((await call({ operation: "put_link", link: { ...link, is_admin: true }, actor: "operator", now })).status, 400); + assert.equal((await call({ operation: "put_link", link: { ...link, org_id: null, team_id: "team:foreign" }, actor: "operator", now })).status, 400); + assert.equal(sql.prepare("SELECT subject_digest FROM saml_subject_links").get().subject_digest, hash("b")); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM saml_audit").get().n, 1); +}); + +test("private target and strict bounded envelopes reject arbitrary operations", async t => { + const { call, env } = await fixture(t); + for (const url of ["https://intelligence.internal/v1/managed-state/saml", "http://other.internal/v1/managed-state/saml", "http://intelligence.internal/v1/managed-state/saml?sql=1"]) + assert.equal((await handleSamlIdentityRequest(request({ operation: "ready" }, url), env)).status, 404); + for (const body of [{ operation: "sql", statement: "DROP TABLE control_users" }, { operation: "ready", extra: true }, + { operation: "audit", issuer_digest: hash("a"), account: null, outcome: "raw assertion", now }, { operation: "list_links", offset: -1 }]) + assert.equal((await call(body)).status, 400); + const oversized = request({ operation: "ready", pad: "x".repeat(9000) }); + assert.equal((await handleSamlIdentityRequest(oversized, env)).status, 400); +}); + +test("the existing Worker forwards SAML URL, method, cookies and token body to the Container", async () => { + const { default: worker } = await loadWorkerModule(); + for (const [path, method, body] of [["/auth/saml/login?next=fixture", "GET", undefined], ["/auth/saml/callback?state=fixture", "POST", "token=synthetic&state=fixture"], ["/auth/saml/metadata", "GET", undefined]]) { + let forwarded; + const response = await worker.fetch(new Request("https://gateway.example" + path, { method, body, headers: { cookie: "session=synthetic", "content-type": "application/x-www-form-urlencoded" } }), { + MULTILLM_PROXY_CONTAINER: { getByName: () => ({ async fetch(req) { forwarded = req; return new Response("container-result", { status: 503 }); } }) } + }, {}); + assert.equal(response.status, 503); assert.equal(await response.text(), "container-result"); + assert.equal(forwarded.url, "https://gateway.example" + path); assert.equal(forwarded.method, method); + assert.equal(forwarded.headers.get("cookie"), "session=synthetic"); assert.equal(await forwarded.text(), body ?? ""); + } +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..7a787730 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,10 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "SAML_ENABLED", + "SAML_BROKER_URL", + "SAML_TRUST_CONFIG_JSON", + "SAML_CALLBACK_URL", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..b91c2c31 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "saml_requests", "saml_subject_links", "saml_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/saml-identity.mjs b/worker/saml-identity.mjs new file mode 100644 index 00000000..0a0d57c7 --- /dev/null +++ b/worker/saml-identity.mjs @@ -0,0 +1,167 @@ +/** Private durable broker request claims, explicit subject links and content-free audit. */ +const PATH = "/v1/managed-state/saml"; +const MAX_BYTES = 8192; +const HASH = /^[a-f0-9]{64}$/; +const ID = /^[a-f0-9]{32}$/; +const OPAQUE = /^[A-Za-z0-9_:.\-]{1,128}$/; +const OUTCOMES = new Set(["authorized", "link_put", "link_deactivated", "saml_assertion_invalid", "saml_assertion_replayed", + "saml_subject_unlinked", "saml_identity_denied", "saml_identity_unavailable", "saml_session_unavailable"]); +const LINK_COLUMNS = "id,issuer_digest,subject_digest,account,org_id,team_id,grants_revision,active,created_at,updated_at"; +const fields = (body, names) => Object.keys(body).length === names.length + 2 + && ["version", "operation", ...names].every(name => Object.hasOwn(body, name)); +const text = value => typeof value === "string" && value.length > 0 && value.length <= 128 && !/[\x00-\x1f\x7f]/.test(value); +const hash = value => typeof value === "string" && HASH.test(value); +const id = value => typeof value === "string" && ID.test(value); +const integer = value => Number.isSafeInteger(value) && value >= 0 && value < Number.MAX_SAFE_INTEGER; +const scope = value => value === null || (typeof value === "string" && OPAQUE.test(value)); +const actorDigest = async value => Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)))) + .map(byte => byte.toString(16).padStart(2, "0")).join(""); +const reply = (value, status = 200) => Response.json(value.error + ? { version: 1, error: { code: value.error, message: "SAML identity storage request could not be completed.", retryable: false } } + : { version: 1, ...value }, { status, headers: { "cache-control": "no-store" } }); +let warned = false; + +function enabled(env) { + const flag = typeof (env.SAML_ENABLED ?? "") === "string" ? (env.SAML_ENABLED ?? "").trim().toLowerCase() : "invalid"; + if (["", "false", "0", "off", "no"].includes(flag)) return false; + if (["true", "1", "on", "yes"].includes(flag)) return true; + if (!warned) { console.warn("Invalid SAML_ENABLED; federation disabled"); warned = true; } + return false; +} + +async function bodyDocument(request) { + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json" || !request.body) + throw new Error("Invalid content type"); + const length = request.headers.get("content-length"); + if (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) throw new Error("Invalid length"); + const reader = request.body.getReader(), chunks = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_BYTES) { void reader.cancel().catch(() => {}); throw new Error("Oversized request"); } + chunks.push(value); + } + } finally { reader.releaseLock(); } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + const body = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + if (!body || typeof body !== "object" || Array.isArray(body) || body.version !== 1) throw new Error("Invalid envelope"); + return body; +} + +function validLink(link) { + return link && typeof link === "object" && !Array.isArray(link) + && Object.keys(link).length === 7 && ["id", "issuer_digest", "subject_digest", "account", "org_id", "team_id", "grants_revision"].every(key => Object.hasOwn(link, key)) + && id(link.id) && hash(link.issuer_digest) && hash(link.subject_digest) && text(link.account) + && scope(link.org_id) && scope(link.team_id) && (link.team_id === null || link.org_id !== null) && integer(link.grants_revision); +} + +function validOperation(body) { + switch (body.operation) { + case "ready": return fields(body, []); + case "create_request": return fields(body, ["state_digest", "nonce_digest", "recipient_digest", "now", "expires_at"]) + && hash(body.state_digest) && hash(body.nonce_digest) && hash(body.recipient_digest) && integer(body.now) && integer(body.expires_at) + && body.expires_at > body.now && body.expires_at <= body.now + 300; + case "claim_request": return fields(body, ["state_digest", "nonce_digest", "recipient_digest", "now"]) + && hash(body.state_digest) && hash(body.nonce_digest) && hash(body.recipient_digest) && integer(body.now); + case "lookup_link": return fields(body, ["issuer_digest", "subject_digest"]) && hash(body.issuer_digest) && hash(body.subject_digest); + case "get_link": return fields(body, ["id"]) && id(body.id); + case "list_links": return fields(body, ["offset"]) && integer(body.offset) && body.offset <= 999999; + case "put_link": return fields(body, ["link", "actor", "now"]) && validLink(body.link) && text(body.actor) && integer(body.now); + case "deactivate_link": return fields(body, ["id", "actor", "now"]) && id(body.id) && text(body.actor) && integer(body.now); + case "audit": return fields(body, ["issuer_digest", "account", "outcome", "now"]) && hash(body.issuer_digest) + && (body.account === null || text(body.account)) && OUTCOMES.has(body.outcome) && integer(body.now); + default: return false; + } +} + +async function ready(db) { + // Verify all columns before any mutation, including the audit destination. + for (const query of ["SELECT state_digest,nonce_digest,recipient_digest,created_at,expires_at,claimed_at FROM saml_requests LIMIT 0", + `SELECT ${LINK_COLUMNS} FROM saml_subject_links LIMIT 0`, "SELECT id,issuer_digest,account,actor_digest,outcome,occurred_at FROM saml_audit LIMIT 0", + "SELECT username,revoked_at FROM control_users LIMIT 0"]) await db.prepare(query).all(); +} + +async function putLink(db, body) { + const link = body.link; + const actor = await actorDigest(body.actor); + const account = await db.prepare("SELECT username FROM control_users WHERE username=? AND revoked_at IS NULL").bind(link.account).first(); + if (!account) return reply({ error: "saml_subject_unlinked" }, 403); + const results = await db.batch([ + db.prepare(`INSERT INTO saml_subject_links(${LINK_COLUMNS}) + SELECT ?,?,?,?,?,?,?,1,?,? FROM control_users WHERE username=? AND revoked_at IS NULL + ON CONFLICT(id) DO UPDATE SET account=excluded.account,org_id=excluded.org_id,team_id=excluded.team_id, + grants_revision=excluded.grants_revision,active=1,updated_at=excluded.updated_at + WHERE saml_subject_links.issuer_digest=excluded.issuer_digest AND saml_subject_links.subject_digest=excluded.subject_digest`) + .bind(link.id, link.issuer_digest, link.subject_digest, link.account, link.org_id, link.team_id, link.grants_revision, body.now, body.now, link.account), + db.prepare(`INSERT INTO saml_audit(id,issuer_digest,account,actor_digest,outcome,occurred_at) + SELECT ?,issuer_digest,account,?,'link_put',? FROM saml_subject_links + WHERE id=? AND issuer_digest=? AND subject_digest=? AND account=? AND updated_at=? AND active=1`) + .bind(crypto.randomUUID(), actor, body.now, link.id, link.issuer_digest, link.subject_digest, link.account, body.now) + ]); + if (results[0].meta.changes !== 1 || results[1].meta.changes !== 1) return reply({ error: "saml_link_conflict" }, 409); + return reply({ link: await db.prepare(`SELECT ${LINK_COLUMNS} FROM saml_subject_links WHERE id=?`).bind(link.id).first() }); +} + +async function deactivateLink(db, body) { + const actor = await actorDigest(body.actor); + const results = await db.batch([ + db.prepare(`INSERT INTO saml_audit(id,issuer_digest,account,actor_digest,outcome,occurred_at) + SELECT ?,issuer_digest,account,?,'link_deactivated',? FROM saml_subject_links WHERE id=? AND active=1`) + .bind(crypto.randomUUID(), actor, body.now, body.id), + db.prepare("UPDATE saml_subject_links SET active=0,updated_at=? WHERE id=? AND active=1").bind(body.now, body.id) + ]); + return reply({ deactivated: results[1].meta.changes === 1 }); +} + +async function execute(db, body) { + switch (body.operation) { + case "ready": return reply({ ready: true }); + case "create_request": { + const result = await db.prepare("INSERT INTO saml_requests(state_digest,nonce_digest,recipient_digest,created_at,expires_at) VALUES(?,?,?,?,?)") + .bind(body.state_digest, body.nonce_digest, body.recipient_digest, body.now, body.expires_at).run(); + return reply({ created: result.meta.changes === 1 }); + } + case "claim_request": { + const result = await db.prepare(`UPDATE saml_requests SET claimed_at=? WHERE state_digest=? AND nonce_digest=? + AND recipient_digest=? AND claimed_at IS NULL AND expires_at>? AND created_at<=?`) + .bind(body.now, body.state_digest, body.nonce_digest, body.recipient_digest, body.now, body.now + 5).run(); + return reply({ claimed: result.meta.changes === 1 }); + } + case "lookup_link": return reply({ link: await db.prepare(`SELECT ${LINK_COLUMNS} FROM saml_subject_links + WHERE issuer_digest=? AND subject_digest=? AND active=1`).bind(body.issuer_digest, body.subject_digest).first() }); + case "get_link": return reply({ link: await db.prepare(`SELECT ${LINK_COLUMNS} FROM saml_subject_links WHERE id=?`).bind(body.id).first() }); + case "list_links": return reply({ links: (await db.prepare(`SELECT ${LINK_COLUMNS} FROM saml_subject_links ORDER BY id LIMIT 100 OFFSET ?`).bind(body.offset).all()).results }); + case "put_link": return putLink(db, body); + case "deactivate_link": return deactivateLink(db, body); + case "audit": { + const result = await db.prepare("INSERT INTO saml_audit(id,issuer_digest,account,outcome,occurred_at) VALUES(?,?,?,?,?)") + .bind(crypto.randomUUID(), body.issuer_digest, body.account, body.outcome, body.now).run(); + return reply({ recorded: result.meta.changes === 1 }); + } + default: return reply({ error: "saml_request_invalid" }, 400); + } +} + +/** Register this handler at handleManagedStateRequest's fixed SAML domain. */ +export async function handleSamlIdentityRequest(request, env) { + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== PATH || url.search || url.hash + || url.username || url.password || request.method !== "POST" || !enabled(env)) return reply({ error: "not_found" }, 404); + let body; + try { body = await bodyDocument(request); } + catch { return reply({ error: "saml_request_invalid" }, 400); } + if (!validOperation(body)) return reply({ error: "saml_request_invalid" }, 400); + if (!env.INTELLIGENCE_DB) return reply({ error: "saml_storage_unavailable" }, 503); + try { + await ready(env.INTELLIGENCE_DB); + return await execute(env.INTELLIGENCE_DB, body); + } catch (error) { + const conflict = /UNIQUE constraint failed/.test(String(error?.message)); + return reply({ error: conflict ? "saml_link_conflict" : "saml_storage_unavailable" }, conflict ? 409 : 503); + } +} From 595399493ad15e362ac31ffbbf86031e3403ea7f Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:28:05 +0530 Subject: [PATCH 2/9] feat(billing): add opt-in Stripe top-ups credited only from verified webhooks PAYMENTS_ENABLED (off by default) adds POST /v1/payments/checkout, which creates a Stripe Checkout Session for a whole-cent USD amount between $0.50 and $1,000 from an allowlisted return origin, with per-principal daily limits and idempotent replay. POST /v1/payments/webhook verifies the raw body's v1 signature and timestamp, claims each event before any callback, and checks livemode, merchant, currency, amount and session against the stored checkout. Credits are added only through the injected payment callback, once per checkout; refunds and disputes append compensation. The browser return page never credits. Configuration holds secret references only (migration 0038). --- docs/payment-billing.md | 165 +++++++ .../0038_payment_billing.sql | 51 ++ package.json | 2 +- routes/payments.py | 71 +++ services/payment_billing.py | 453 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 5 + tests/test_payment_billing.py | 393 +++++++++++++++ tests/test_payment_billing_worker.mjs | 219 +++++++++ worker/container-env.mjs | 3 + worker/d1-schema.mjs | 2 +- worker/payment-webhooks.mjs | 250 ++++++++++ 11 files changed, 1612 insertions(+), 2 deletions(-) create mode 100644 docs/payment-billing.md create mode 100644 intelligence-migrations/0038_payment_billing.sql create mode 100644 routes/payments.py create mode 100644 services/payment_billing.py create mode 100644 tests/test_payment_billing.py create mode 100644 tests/test_payment_billing_worker.mjs create mode 100644 worker/payment-webhooks.mjs diff --git a/docs/payment-billing.md b/docs/payment-billing.md new file mode 100644 index 00000000..22356db9 --- /dev/null +++ b/docs/payment-billing.md @@ -0,0 +1,165 @@ +# Top-ups and payment billing + +Payments create explicit USD top-ups through Stripe Checkout. A browser redirect +has no effect on a balance. Credits come only from signed processor evidence and +the configured credits ledger callback. + +## Configuration + +Payments are disabled by default. With `PAYMENTS_ENABLED` unset, empty or false, +every `/v1/payments/*` request returns JSON 404 without payment storage or +processor access. A malformed flag or configuration disables payments and emits +one warning without configuration values. + +An enabled example uses references to privately configured secrets: + +```json +{ + "PAYMENTS_ENABLED": "true", + "PAYMENT_PROCESSOR_CONFIG_JSON": "{\"processor\":\"stripe\",\"secret_key_ref\":\"STRIPE_MERCHANT_SECRET\",\"return_origins\":[\"https://billing.example.com\"],\"product_name\":\"Gateway credits\"}", + "PAYMENT_WEBHOOK_KEY_REF": "STRIPE_ENDPOINT_SECRET" +} +``` + +The merchant key and webhook signing secret are supplied by the host's secret +resolver. Their values never belong in the JSON configuration, responses or +logs. Use a webhook endpoint belonging to the same direct Stripe merchant as +the Checkout key. Connected-account events are rejected. Test and live mode +must match the Checkout Session returned by Stripe. + +Payment routes require a migrated durable store. The private D1 domain is +`http://intelligence.internal/v1/managed-state/payments`, with fixed reserve, +save, find, claim and finish operations. `D1PaymentStore` accepts a private +transport; `SqlPaymentStore` accepts an existing migrated SQLite database. +Neither creates tables or falls back to memory. Missing tables, missing +columns, unavailable storage or unconnected storage return JSON 503 +`payments_unavailable` before any processor request. Apply the additive payment +migration through the normal migration process before enabling the flag. + +## Checkout + +Send a gateway key to `POST /v1/payments/checkout` with: + +```json +{ + "amount_microusd": 500000, + "currency": "USD", + "idempotency_key": "topup-2026-10-09-001", + "return_url": "https://billing.example.com/return" +} +``` + +The default permission grants only the verified key owner in the legacy +single-principal scope. Organisation workspaces require the injected permission +authority to verify a billing or admin role and an injected verified tenant +context resolver. Client body fields cannot select a workspace. Gateway key +scope enforcement also applies. + +Amounts are integer micro-USD, in whole cents: multiples of 10,000, from 500,000 +($0.50) to 1,000,000,000 ($1,000). The currency is exactly `USD`. The return URL +must use HTTPS and match an allowlisted origin, including its port, exactly. +An idempotency key is an opaque 1–128 character identifier using letters, +digits, underscores, colons, periods or hyphens. + +At most ten distinct checkout reservations are accepted per principal per UTC +day, including processor failures. Reusing the same scoped idempotency key and +body returns the same checkout without consuming another reservation. Reusing +it with a different body returns 409 `payment_idempotency_conflict`. Concurrent +creation in another process can return retryable 503 +`payment_checkout_processing`; retry the same key and body. + +The processor request is form encoded, goes only to +`https://api.stripe.com/v1/checkout/sessions`, has redirects disabled and uses +the opaque checkout ID as Stripe's idempotency key and client reference. Its +only metadata is that opaque ID. Processor failures return 503 +`payment_processor_unavailable`; retrying preserves the original processor +idempotency ID. + +A successful checkout response is: + +```json +{ + "checkout_id": "pay_example", + "url": "https://checkout.stripe.com/c/example", + "amount_microusd": 500000, + "currency": "USD", + "status": "pending" +} +``` + +Opening the URL allows payment at Stripe. Returning to the browser URL never +credits a balance. The returned `pending` describes the checkout response, +not a confirmed payment. + +## Processor evidence and reconciliation + +Configure Stripe to deliver these types to `POST /v1/payments/webhook`, and +confirm them in the Stripe dashboard: + +- `checkout.session.completed`: credit only with `payment_status=paid`. +- `checkout.session.async_payment_succeeded`: credit the delayed payment. +- `checkout.session.async_payment_failed`: record failure without credit. +- `charge.refunded`: append the increase in cumulative refunded amount. +- `charge.dispute.created`: append a dispute compensation or hold through the + ledger callback, using the dispute ID as its stable operation identity. + +The webhook requires no gateway key and is CSRF-exempt. It reads raw bytes +before parsing JSON, verifies every supported `v1` signature candidate with +HMAC-SHA256 and requires the timestamp to be within 300 seconds in either +direction. Re-serialization invalidates signatures. Unsupported schemes never +authorize an event. Bodies are limited to 65,536 bytes. + +Signed events must match the stored session, direct merchant, mode, currency and +amount. Refunds and disputes must link through the payment intent recorded on +verified session evidence. Refund totals cannot exceed the original charge; +disputes cannot exceed the checkout amount. Mismatches and unknown types are +recorded and acknowledged without credit. + +Refund or dispute evidence arriving before the session's payment intent is known +is retained as `pending_match` and returns 503 for retry. After the verified +session links the intent, the same event can be reconciled without losing its +original evidence hash or audit history. + +An atomic event claim precedes the immutable `PaymentEvent` callback. A +completed replay returns 200 without invoking it again. Both session success +types share one credit operation, cumulative refund notifications apply only +their delta, and repeat dispute IDs share one compensation operation. An event +ID reused with different bytes returns 400 `payment_event_conflict`. + +The callback is the only credit authority. It must apply its immutable +idempotency ID atomically, including when recovering from a process crash after +ledger application but before the payment record is finalized. A missing, +denied, failing or unbound callback leaves `pending_credit` evidence and returns +503 `payment_credit_unavailable`, allowing Stripe to retry. Concurrent active +claims also return this error. A claim can be recovered after a 60-second lease; +retry the original event first before processing other events for that checkout. +Do not acknowledge outstanding credit work manually as paid. Ledger history is +never erased by refunds or disputes. + +## Errors, retention and billing policy + +Invalid checkout input or signature returns 400; absent authentication returns +401; denied billing permission returns 403; disabled payments return 404; +idempotency conflicts return 409; velocity exhaustion returns 429 +`payment_velocity_limited`; unavailable processor, permission, storage or credit +authority returns 503. Payment responses use `Cache-Control: no-store` and +contain no secrets. Unknown payment methods return 405 when enabled. + +Stored records contain opaque identity and checkout scope, processor IDs, +integer amounts, mode, event/body hashes, claims, revisions and content-free +audit outcomes. Return URLs and raw webhook bodies are not stored. Card data, +customer names and email addresses are not stored. Operational records have no +automatic purge; retain them according to reconciliation policy. Velocity +counters use UTC day partitions and are independent of refund history. + +Taxes, invoices, processor fees and charge policies remain explicit operator +settings. The top-up amount is the credit amount; estimated usage is not an +invoice. Rehearse the configured checkout and webhook flow in the merchant +sandbox before accepting live payments. + +The fixed processor contract follows the Stripe documentation: +[webhooks](https://docs.stripe.com/webhooks), +[signature verification](https://docs.stripe.com/webhooks/signature), +[Checkout Session creation](https://docs.stripe.com/api/checkout/sessions/create), +[Checkout metadata](https://docs.stripe.com/payments/checkout-sessions), and +[currency units](https://docs.stripe.com/glossary). Verify endpoint event selection in the Stripe dashboard. diff --git a/intelligence-migrations/0038_payment_billing.sql b/intelligence-migrations/0038_payment_billing.sql new file mode 100644 index 00000000..1baf823b --- /dev/null +++ b/intelligence-migrations/0038_payment_billing.sql @@ -0,0 +1,51 @@ +CREATE TABLE IF NOT EXISTS payment_checkouts ( + checkout_id TEXT PRIMARY KEY, + principal TEXT NOT NULL, + context_json TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + body_hash TEXT NOT NULL, + amount_microusd INTEGER NOT NULL CHECK(amount_microusd BETWEEN 500000 AND 1000000000 AND amount_microusd % 10000 = 0), + currency TEXT NOT NULL CHECK(currency = 'USD'), + status TEXT NOT NULL DEFAULT 'creating', + checkout_url TEXT, + session_id TEXT UNIQUE, + payment_intent TEXT UNIQUE, + merchant TEXT, + livemode INTEGER, + credited INTEGER NOT NULL DEFAULT 0, + refunded INTEGER NOT NULL DEFAULT 0, + revision INTEGER NOT NULL DEFAULT 0, + claim_token TEXT, + claim_until INTEGER NOT NULL DEFAULT 0, + active_event TEXT, + created_at INTEGER NOT NULL, + UNIQUE(principal, context_json, idempotency_key) +); +CREATE TABLE IF NOT EXISTS payment_events ( + event_id TEXT PRIMARY KEY, + evidence_hash TEXT NOT NULL, + checkout_id TEXT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + amount INTEGER NOT NULL DEFAULT 0, + target_amount INTEGER NOT NULL DEFAULT 0, + operation_id TEXT, + revision INTEGER NOT NULL DEFAULT 0, + claim_token TEXT, + claim_until INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS payment_velocity ( + principal TEXT NOT NULL, + utc_day TEXT NOT NULL, + attempts INTEGER NOT NULL CHECK(attempts BETWEEN 0 AND 10), + PRIMARY KEY(principal, utc_day) +); +CREATE TABLE IF NOT EXISTS payment_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + checkout_id TEXT, + event_id TEXT, + outcome TEXT NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS payment_events_checkout ON payment_events(checkout_id, created_at); diff --git a/package.json b/package.json index c66e4a8f..139b5d3f 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_payment_billing_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/payments.py b/routes/payments.py new file mode 100644 index 00000000..45c061bc --- /dev/null +++ b/routes/payments.py @@ -0,0 +1,71 @@ +"""Authenticated checkout and raw, CSRF-exempt processor evidence.""" +import json + +from flask import g, jsonify, request + +from route_helpers import api_authenticate_only +from services import payment_billing as payments +from services.enterprise_contract import TenantContext + + +def register_payment_routes(app, csrf, *, service=None, context_resolver=None): + if app.extensions.get("payment_routes_registered"): + return + service = service or payments.PaymentBilling() + app.extensions["payment_billing"] = service + app.extensions["payment_routes_registered"] = True + + def failed(error): + return jsonify({"error": {"code": error.code, "message": "Payment operation unavailable."}}), error.status + + def payment_gate(): + if not request.path.startswith("/v1/payments/"): + return None + try: + service.config() + except payments.PaymentError as error: + return failed(error) + if request.content_length is not None and request.content_length > payments.MAX_BODY: + return failed(payments.PaymentError("invalid_payment_body", 400)) + return None + + app.before_request_funcs.setdefault(None, []).insert(0, payment_gate) + + @app.after_request + def payment_headers(response): + if request.path.startswith("/v1/payments/"): + response.headers["Cache-Control"] = "no-store" + return response + + @app.post("/v1/payments/checkout") + @csrf.exempt + @api_authenticate_only(required_scope="models") + def payment_checkout(): + try: + user = g.authenticated_user + owner = str(user.get("id") or user.get("username") or "") + context = context_resolver(user) if context_resolver else TenantContext(owner) + if not request.is_json: + raise payments.PaymentError("invalid_payment_checkout", 400) + raw = request.stream.read(payments.MAX_BODY + 1) + if len(raw) > payments.MAX_BODY: + raise payments.PaymentError("invalid_payment_body", 400) + try: + body = json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + raise payments.PaymentError("invalid_payment_checkout", 400) from None + return jsonify(service.checkout(context, owner, body)) + except payments.PaymentError as error: + return failed(error) + except (ValueError, TypeError): + return failed(payments.PaymentError("payment_permission_denied", 403)) + + @app.post("/v1/payments/webhook") + @csrf.exempt + def payment_webhook(): + try: + # Signature validation always precedes parsing or reading identity. + raw = request.stream.read(payments.MAX_BODY + 1) + return jsonify(service.webhook(raw, request.headers.get("Stripe-Signature", ""))) + except payments.PaymentError as error: + return failed(error) diff --git a/services/payment_billing.py b/services/payment_billing.py new file mode 100644 index 00000000..26704304 --- /dev/null +++ b/services/payment_billing.py @@ -0,0 +1,453 @@ +"""Fixed Stripe checkout transport and verified, durable payment evidence.""" +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import re +import sqlite3 +import threading +import time +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from urllib.parse import urlencode, urlsplit + +import requests + +from services.enterprise_contract import (PaymentCallback, PaymentEvent, TenantContext, deliver_payment, + register_enterprise_adapters) + +logger = logging.getLogger(__name__) +ENDPOINT = "https://api.stripe.com/v1/checkout/sessions" +MAX_BODY = 65536 +LEASE_SECONDS = 60 +LABEL = re.compile(r"[A-Za-z0-9_:.-]{1,128}\Z") +REF = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}\Z") +SCHEMA_QUERIES = ( + "SELECT checkout_id,principal,context_json,idempotency_key,body_hash,amount_microusd,currency,status,checkout_url,session_id,payment_intent,merchant,livemode,credited,refunded,revision,claim_token,claim_until,active_event,created_at FROM payment_checkouts LIMIT 0", + "SELECT event_id,evidence_hash,checkout_id,kind,status,amount,target_amount,operation_id,revision,claim_token,claim_until,created_at FROM payment_events LIMIT 0", + "SELECT principal,utc_day,attempts FROM payment_velocity LIMIT 0", + "SELECT id,checkout_id,event_id,outcome,created_at FROM payment_audit LIMIT 0") + + +class PaymentError(Exception): + def __init__(self, code="payments_unavailable", status=503): + super().__init__(code) + self.code, self.status = code, status + + +@lru_cache(maxsize=2) +def _warn_once(kind): + logger.warning("Invalid payment %s; payments disabled", kind) + + +@dataclass(frozen=True) +class ProcessorConfig: + secret_key_ref: str + webhook_key_ref: str + return_origins: tuple[str, ...] + product_name: str + + +def origin(url): + if not isinstance(url, str) or len(url) > 2048 or any(ord(c) <= 32 or ord(c) == 127 for c in url): + raise ValueError("Invalid URL") + parsed = urlsplit(url) + if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or "\\" in url: + raise ValueError("Invalid URL") + return f"https://{parsed.hostname.lower()}" + (f":{parsed.port}" if parsed.port not in (None, 443) else "") + + +def settings(env) -> ProcessorConfig | None: + raw = env.get("PAYMENTS_ENABLED", "") + flag = raw.strip().lower() if isinstance(raw, str) else "invalid" if raw is not None else "" + if flag in {"", "0", "false", "off", "no"}: + return None + if flag not in {"1", "true", "on", "yes"}: + _warn_once("flag") + return None + try: + value = json.loads(env.get("PAYMENT_PROCESSOR_CONFIG_JSON") or "{}") + webhook = env.get("PAYMENT_WEBHOOK_KEY_REF", "") + if (type(value) is not dict or set(value) != {"processor", "secret_key_ref", "return_origins", "product_name"} + or value["processor"] != "stripe" or not isinstance(webhook, str) or not REF.fullmatch(webhook) + or not isinstance(value["secret_key_ref"], str) or not REF.fullmatch(value["secret_key_ref"]) + or not isinstance(value["product_name"], str) or not 1 <= len(value["product_name"]) <= 128 + or any(ord(c) < 32 for c in value["product_name"]) + or type(value["return_origins"]) is not list or not 1 <= len(value["return_origins"]) <= 16): + raise ValueError() + origins = tuple(origin(url) for url in value["return_origins"]) + if any(url != normalized for url, normalized in zip(value["return_origins"], origins)): + raise ValueError() + return ProcessorConfig(value["secret_key_ref"], webhook, origins, value["product_name"]) + except (ValueError, TypeError, KeyError): + _warn_once("configuration") + return None + + +def verify_signature(raw: bytes, header: str, secret: str, now: int) -> bool: + try: + if not isinstance(raw, bytes) or len(raw) > MAX_BODY or not secret or len(header) > 8192: + return False + pieces = [part.strip().split("=", 1) for part in header.split(",")] + timestamps = [value for key, value in pieces if key == "t"] + signatures = [value for key, value in pieces if key == "v1"] + if len(timestamps) != 1 or not re.fullmatch(r"[0-9]{1,12}", timestamps[0]): + return False + timestamp = timestamps[0] + if abs(now - int(timestamp)) > 300: + return False + expected = hmac.new(secret.encode(), timestamp.encode() + b"." + raw, hashlib.sha256).hexdigest() + return any(hmac.compare_digest(expected, candidate) for candidate in signatures + if re.fullmatch(r"[0-9a-f]{64}", candidate)) + except (TypeError, ValueError, AttributeError, UnicodeError): + return False + + +def owner_permission(context, owner): + return context.principal_id == owner and context.org_id is None and context.team_id is None + + +def checked_checkout(body, config): + if (type(body) is not dict or set(body) != {"amount_microusd", "currency", "idempotency_key", "return_url"} + or type(body["amount_microusd"]) is not int or not 500000 <= body["amount_microusd"] <= 1000000000 + or body["amount_microusd"] % 10000 or body["currency"] != "USD" + or not isinstance(body["idempotency_key"], str) or not LABEL.fullmatch(body["idempotency_key"])): + raise PaymentError("invalid_payment_checkout", 400) + try: + if origin(body["return_url"]) not in config.return_origins: + raise ValueError() + except (ValueError, TypeError): + raise PaymentError("invalid_payment_checkout", 400) from None + return hashlib.sha256(json.dumps(body, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def public_checkout(row): + return {"checkout_id": row["checkout_id"], "url": row["checkout_url"], + "amount_microusd": row["amount_microusd"], "currency": "USD", "status": "pending"} + + +def stripe_http(url, *, headers, body): + """The only processor transport; redirects never receive merchant credentials.""" + if url != ENDPOINT: + raise PaymentError() + response = requests.post(ENDPOINT, headers=headers, data=body, timeout=(3, 10), allow_redirects=False) + if response.status_code != 200: + raise PaymentError("payment_processor_unavailable", 503) + return response.json() + + +class SqlPaymentStore: + """A migrated SQLite authority; no runtime schema creation or memory fallback.""" + def __init__(self, path): + self.path = path + + @contextmanager + def transaction(self): + db = None + try: + db = sqlite3.connect(Path(self.path).resolve().as_uri() + "?mode=rw", uri=True, timeout=10) + db.row_factory = sqlite3.Row + for query in SCHEMA_QUERIES: + db.execute(query) + db.execute("BEGIN IMMEDIATE") + yield db + db.commit() + except sqlite3.Error: + if db is not None: + db.rollback() + raise PaymentError() from None + finally: + if db is not None: + db.close() + + def call(self, operation, **data): + with self.transaction() as db: + handler = getattr(self, "_" + operation, None) + if operation not in {"reserve", "save", "find", "claim", "finish"} or handler is None: + raise PaymentError("invalid_payment_operation", 400) + return handler(db, **data) + + def _reserve(self, db, *, context, body_hash, idempotency_key, checkout_id, amount, now, token): + scope = json.dumps(context, sort_keys=True, separators=(",", ":")) + row = db.execute("SELECT * FROM payment_checkouts WHERE principal=? AND context_json=? AND idempotency_key=?", + (context["principal_id"], scope, idempotency_key)).fetchone() + if row: + if row["body_hash"] != body_hash: + raise PaymentError("payment_idempotency_conflict", 409) + if not row["checkout_url"] and row["claim_until"] <= now: + db.execute("UPDATE payment_checkouts SET claim_token=?,claim_until=? WHERE checkout_id=?", + (token, now + LEASE_SECONDS, row["checkout_id"])) + else: + day = datetime.fromtimestamp(now, timezone.utc).strftime("%Y-%m-%d") + db.execute("INSERT INTO payment_velocity VALUES (?,?,0) ON CONFLICT DO NOTHING", (context["principal_id"], day)) + changed = db.execute("UPDATE payment_velocity SET attempts=attempts+1 WHERE principal=? AND utc_day=? AND attempts<10", + (context["principal_id"], day)).rowcount + if not changed: + raise PaymentError("payment_velocity_limited", 429) + db.execute("INSERT INTO payment_checkouts (checkout_id,principal,context_json,idempotency_key,body_hash,amount_microusd,currency,claim_token,claim_until,created_at) VALUES (?,?,?,?,?,?,'USD',?,?,?)", + (checkout_id, context["principal_id"], scope, idempotency_key, body_hash, amount, token, now+LEASE_SECONDS, now)) + db.execute("INSERT INTO payment_audit (checkout_id,outcome,created_at) VALUES (?,'created',?)", (checkout_id, now)) + row = db.execute("SELECT * FROM payment_checkouts WHERE principal=? AND context_json=? AND idempotency_key=?", + (context["principal_id"], scope, idempotency_key)).fetchone() + return dict(row) + + def _save(self, db, *, checkout_id, token, session_id=None, url=None, livemode=None, merchant=None): + if session_id is None: + db.execute("UPDATE payment_checkouts SET claim_until=0 WHERE checkout_id=? AND claim_token=?", (checkout_id, token)) + else: + changed = db.execute("UPDATE payment_checkouts SET session_id=?,checkout_url=?,livemode=?,merchant=?,status='pending',claim_until=0,claim_token=NULL WHERE checkout_id=? AND claim_token=?", + (session_id, url, int(livemode), merchant, checkout_id, token)).rowcount + if not changed: + raise PaymentError("payment_checkout_processing", 503) + return None + + def _find(self, db, *, checkout_id=None, payment_intent=None): + row = db.execute("SELECT * FROM payment_checkouts WHERE checkout_id=?" if checkout_id else + "SELECT * FROM payment_checkouts WHERE payment_intent=?", (checkout_id or payment_intent,)).fetchone() + return dict(row) if row else None + + def _claim(self, db, *, event_id, evidence_hash, checkout_id, kind, status, amount, target_amount, + operation_id, payment_intent, now, token): + existing = db.execute("SELECT * FROM payment_events WHERE event_id=?", (event_id,)).fetchone() + if existing: + if existing["evidence_hash"] != evidence_hash: + raise PaymentError("payment_event_conflict", 400) + if existing["status"] not in {"processing", "pending_credit", "pending_match"}: + return {**dict(existing), "claimed": False} + if existing["status"] == "pending_match": + if status == "pending_match": + return {**dict(existing), "claimed": False} + existing = None + if existing and existing["claim_until"] > now: + raise PaymentError("payment_credit_unavailable", 503) + row = db.execute("SELECT * FROM payment_checkouts WHERE checkout_id=?", (checkout_id,)).fetchone() + if status == "processing" and row: + if row["active_event"] and row["active_event"] != event_id: + active = db.execute("SELECT claim_until FROM payment_events WHERE event_id=?", (row["active_event"],)).fetchone() + # Retry the original event first, preserving its immutable callback. + if active: + raise PaymentError("payment_credit_unavailable", 503) + if not existing: + prior = db.execute("SELECT status FROM payment_events WHERE operation_id=? AND status IN ('credited','refunded','disputed') LIMIT 1", + (operation_id,)).fetchone() + if prior or (kind == "credit" and row["credited"]): + status = "ignored" + if kind != "credit" and not row["credited"]: + raise PaymentError("payment_credit_unavailable", 503) + if kind == "refund": + amount = max(0, target_amount - row["refunded"]) + if amount == 0: + status = "ignored" + revision = existing["revision"] if existing else row["revision"] if row else 0 + if not existing: + db.execute("INSERT INTO payment_events (event_id,evidence_hash,checkout_id,kind,status,amount,target_amount,operation_id,revision,claim_token,claim_until,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET checkout_id=excluded.checkout_id,kind=excluded.kind,status=excluded.status,amount=excluded.amount,target_amount=excluded.target_amount,operation_id=excluded.operation_id,revision=excluded.revision,claim_token=excluded.claim_token,claim_until=excluded.claim_until WHERE payment_events.status='pending_match' AND payment_events.evidence_hash=excluded.evidence_hash", + (event_id, evidence_hash, checkout_id, kind, status, amount, target_amount, operation_id, revision, + token, now+LEASE_SECONDS if status == "processing" else 0, now)) + db.execute("INSERT INTO payment_audit (checkout_id,event_id,outcome,created_at) VALUES (?,?,?,?)", + (checkout_id, event_id, status, now)) + else: + db.execute("UPDATE payment_events SET status='processing',claim_token=?,claim_until=? WHERE event_id=?", (token, now+LEASE_SECONDS, event_id)) + if status == "processing": + db.execute("UPDATE payment_checkouts SET active_event=?,payment_intent=COALESCE(payment_intent,?) WHERE checkout_id=?", + (event_id, payment_intent, checkout_id)) + elif status == "failed" and row and not row["credited"]: + db.execute("UPDATE payment_checkouts SET status='failed' WHERE checkout_id=?", (checkout_id,)) + result = dict(db.execute("SELECT * FROM payment_events WHERE event_id=?", (event_id,)).fetchone()) + return {**result, "claimed": status == "processing"} + + def _finish(self, db, *, event_id, token, status, now): + row = db.execute("SELECT * FROM payment_events WHERE event_id=? AND claim_token=? AND status='processing'", (event_id, token)).fetchone() + if not row: + raise PaymentError("payment_credit_unavailable", 503) + if status != "pending_credit" and status != {"credit": "credited", "refund": "refunded", "dispute": "disputed"}[row["kind"]]: + raise PaymentError("invalid_payment_operation", 400) + db.execute("UPDATE payment_events SET status=?,claim_until=0 WHERE event_id=?", (status, event_id)) + if status != "pending_credit": + db.execute("UPDATE payment_checkouts SET active_event=NULL,revision=revision+1,credited=CASE WHEN ?='credit' THEN 1 ELSE credited END,refunded=CASE WHEN ?='refund' THEN ? ELSE refunded END,status=? WHERE checkout_id=? AND active_event=?", + (row["kind"], row["kind"], row["target_amount"], status, row["checkout_id"], event_id)) + db.execute("INSERT INTO payment_audit (checkout_id,event_id,outcome,created_at) VALUES (?,?,?,?)", (row["checkout_id"], event_id, status, now)) + return None + + +class D1PaymentStore: + """Injected private transport returns the same fixed domain operations as SQL.""" + def __init__(self, transport): + self.transport = transport + + def call(self, operation, **data): + try: + reply = self.transport({"version": 1, "operation": operation, **data}) + if not isinstance(reply, dict) or set(reply) != {"version", "result"} or reply["version"] != 1: + raise PaymentError() + return reply["result"] + except PaymentError: + raise + except Exception: + raise PaymentError() from None + + +class PaymentBilling: + def __init__(self, *, env=None, store=None, http=None, secret_resolver=None, callback: PaymentCallback | None = None, + permission=owner_permission, clock=time.time): + # Configuration and secret access are supplied by the host, never file reads. + import os + self.env = os.environ if env is None else env + self.store, self.http, self.callback = store, http or stripe_http, callback + self.secret_resolver = secret_resolver or self.env.get + self.permission, self.clock = permission, clock + self.lock = threading.RLock() + + def config(self): + config = settings(self.env) + if config is None: + raise PaymentError("not_found", 404) + return config + + def state(self, operation, **values): + if self.store is None: + raise PaymentError() + return self.store.call(operation, **values) + + def checkout(self, context, owner, body): + config = self.config() + try: + allowed = type(context) is TenantContext and self.permission(context, owner) is True + except Exception: + raise PaymentError("payment_permission_unavailable", 503) from None + if not allowed: + raise PaymentError("payment_permission_denied", 403) + body_hash = checked_checkout(body, config) + with self.lock: + token, now = uuid.uuid4().hex, int(self.clock()) + row = self.state("reserve", context=asdict(context), body_hash=body_hash, + idempotency_key=body["idempotency_key"], checkout_id="pay_"+uuid.uuid4().hex, + amount=body["amount_microusd"], now=now, token=token) + if row["checkout_url"]: + return public_checkout(row) + if row["claim_token"] != token: + raise PaymentError("payment_checkout_processing", 503) + try: + session = self.create_session(config, row, body) + self.state("save", checkout_id=row["checkout_id"], token=token, session_id=session["id"], + url=session["url"], livemode=session["livemode"], merchant="direct") + except Exception: + self.state("save", checkout_id=row["checkout_id"], token=token) + raise PaymentError("payment_processor_unavailable", 503) from None + return public_checkout({**row, "checkout_url": session["url"]}) + + def create_session(self, config, row, body): + secret = self.secret_resolver(config.secret_key_ref) + if not secret or self.http is None: + raise PaymentError() + form = {"mode": "payment", "line_items[0][price_data][currency]": "usd", + "line_items[0][price_data][unit_amount]": str(row["amount_microusd"] // 10000), + "line_items[0][price_data][product_data][name]": config.product_name, + "line_items[0][quantity]": "1", "success_url": body["return_url"], "cancel_url": body["return_url"], + "client_reference_id": row["checkout_id"], "metadata[checkout_id]": row["checkout_id"]} + session = self.http(ENDPOINT, headers={"Authorization": "Bearer " + secret, + "Content-Type": "application/x-www-form-urlencoded", "Idempotency-Key": row["checkout_id"]}, body=urlencode(form)) + if (type(session) is not dict or session.get("object") != "checkout.session" + or not isinstance(session.get("id"), str) or not LABEL.fullmatch(session["id"]) + or type(session.get("amount_total")) is not int or session["amount_total"]*10000 != row["amount_microusd"] + or session.get("currency") != "usd" or type(session.get("livemode")) is not bool + or origin(session.get("url")) != "https://checkout.stripe.com"): + raise PaymentError() + return session + + def webhook(self, raw, signature): + config = self.config() + try: + secret = self.secret_resolver(config.webhook_key_ref) + except Exception: + raise PaymentError() from None + if not secret: + raise PaymentError() + now = int(self.clock()) + if not verify_signature(raw, signature, secret, now): + raise PaymentError("invalid_payment_signature", 400) + try: + value = json.loads(raw) + if type(value) is not dict or value.get("object") != "event" or not LABEL.fullmatch(value.get("id", "")): + raise ValueError() + evidence_hash = hashlib.sha256(raw).hexdigest() + except (ValueError, TypeError, UnicodeError, RecursionError): + raise PaymentError("invalid_payment_event", 400) from None + with self.lock: + row, decision = self.classify(value) + token = uuid.uuid4().hex + claimed = self.state("claim", event_id=value["id"], evidence_hash=evidence_hash, + checkout_id=row["checkout_id"] if row else None, now=now, token=token, **decision) + if not claimed["claimed"]: + if claimed["status"] == "pending_match": + raise PaymentError("payment_credit_unavailable", 503) + return {"received": True, "status": claimed["status"]} + event = PaymentEvent(TenantContext(**json.loads(row["context_json"])), row["checkout_id"], claimed["revision"], + claimed["operation_id"], "stripe", value["id"], claimed["kind"], "USD", claimed["amount"], True) + try: + deliver_payment(register_enterprise_adapters(payment=self.callback), event) + except Exception: + self.state("finish", event_id=value["id"], token=token, status="pending_credit", now=now) + raise PaymentError("payment_credit_unavailable", 503) from None + status = {"credit": "credited", "refund": "refunded", "dispute": "disputed"}[event.kind] + self.state("finish", event_id=value["id"], token=token, status=status, now=now) + return {"received": True, "status": status} + + def classify(self, value): + kind = value.get("type") + decision = {"kind": "ignored", "status": "ignored", "amount": 0, "target_amount": 0, + "operation_id": None, "payment_intent": None} + kinds = {"checkout.session.completed": "credit", "checkout.session.async_payment_succeeded": "credit", + "checkout.session.async_payment_failed": "failed", "charge.refunded": "refund", "charge.dispute.created": "dispute"} + if not isinstance(kind, str) or kind not in kinds: + return None, decision + obj = value.get("data", {}).get("object", {}) if type(value.get("data")) is dict else {} + if type(obj) is not dict: + return None, {**decision, "status": "mismatch"} + session = kind.startswith("checkout.session.") + checkout_id = obj.get("client_reference_id") if session else None + intent = obj.get("payment_intent") + if not isinstance(intent, str) or not LABEL.fullmatch(intent): + intent = None + if (session and (not isinstance(checkout_id, str) or not LABEL.fullmatch(checkout_id))) or (not session and intent is None): + return None, {**decision, "status": "mismatch"} + row = self.state("find", checkout_id=checkout_id, payment_intent=intent) + if row is None and not session: + return None, {**decision, "status": "pending_match"} + amount = obj.get("amount_total") if session else obj.get("amount") + metadata = obj.get("metadata", {}) + mismatch = (row is None or obj.get("currency") != "usd" or type(amount) is not int + or type(value.get("livemode")) is not bool or value["livemode"] != bool(row["livemode"]) + or type(obj.get("livemode")) is not bool or obj["livemode"] != bool(row["livemode"]) + or (value.get("account") or "direct") != row["merchant"]) + if not mismatch and session: + mismatch = (amount*10000 != row["amount_microusd"] or obj.get("id") != row["session_id"] + or type(metadata) is not dict or metadata.get("checkout_id") != checkout_id + or (row["payment_intent"] is not None and intent != row["payment_intent"])) + if not mismatch and not session: + mismatch = (amount <= 0 or amount*10000 > row["amount_microusd"] + or (kind == "charge.refunded" and amount*10000 != row["amount_microusd"])) + if mismatch: + return row, {**decision, "status": "mismatch"} + mapped = kinds[kind] + if kind == "checkout.session.completed" and obj.get("payment_status") != "paid": + return row, decision + if mapped == "failed": + return row, {**decision, "status": "failed"} + target = 0 + if mapped == "refund": + target = obj.get("amount_refunded") + if type(target) is not int or not 0 < target <= amount: + return row, {**decision, "status": "mismatch"} + target *= 10000 + object_id = obj.get("id") + if mapped == "dispute" and (not isinstance(object_id, str) or not LABEL.fullmatch(object_id)): + return row, {**decision, "status": "mismatch"} + operation = f"credit:{row['checkout_id']}" if mapped == "credit" else f"refund:{row['checkout_id']}:{target}" if mapped == "refund" else "dispute:" + hashlib.sha256(object_id.encode()).hexdigest() + return row, {"kind": mapped, "status": "processing", "amount": amount*10000, + "target_amount": target, "operation_id": operation, "payment_intent": intent} diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..488ebffd 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -145,6 +145,10 @@ "routes/optimized.py": [ "app.route('/optimize/v1/chat/completions', methods=['POST', 'OPTIONS'])" ], + "routes/payments.py": [ + "app.post('/v1/payments/checkout')", + "app.post('/v1/payments/webhook')" + ], "routes/prompt_templates.py": [ "app.route('/admin/workbench/prompts', methods=['GET', 'POST'])", "app.route('/v1/prompt-templates//render', methods=['POST', 'OPTIONS'])" @@ -340,6 +344,7 @@ "worker/opencode/reasoning-policy.mjs", "worker/opencode/reasoning-request.mjs", "worker/opencode/reasoning-response.mjs", + "worker/payment-webhooks.mjs", "worker/pii-redaction.mjs", "worker/pii-rehydration.mjs", "worker/prometheus.mjs", diff --git a/tests/test_payment_billing.py b/tests/test_payment_billing.py new file mode 100644 index 00000000..e01c4268 --- /dev/null +++ b/tests/test_payment_billing.py @@ -0,0 +1,393 @@ +"""Offline checkout, signed evidence, durable claims and authenticated routes.""" +import hashlib +import hmac +import json +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import parse_qs + +import pytest +from flask import Flask, g + +from services import payment_billing as pb +from services.enterprise_contract import AuthorityResult, TenantContext +from routes import payments + +NOW = 1700000000 +SECRET = "public-test-webhook-secret" +VECTOR = b'{"id":"evt_vector","object":"event","type":"unknown","livemode":false}' +SIGNATURE = hmac.new(SECRET.encode(), str(NOW).encode() + b"." + VECTOR, hashlib.sha256).hexdigest() +ROOT = Path(__file__).resolve().parents[1] +CONFIG = {"processor": "stripe", "secret_key_ref": "PAYMENT_TEST_KEY", + "return_origins": ["https://portal.test"], "product_name": "Gateway credits"} +ENV = {"PAYMENTS_ENABLED": "true", "PAYMENT_PROCESSOR_CONFIG_JSON": json.dumps(CONFIG), + "PAYMENT_WEBHOOK_KEY_REF": "PAYMENT_TEST_WEBHOOK"} +BODY = {"amount_microusd": 500000, "currency": "USD", "idempotency_key": "order-1", + "return_url": "https://portal.test/paid"} + + +@pytest.fixture +def setup(tmp_path): + path = tmp_path / "payments.sqlite" + with sqlite3.connect(path) as db: + db.execute("CREATE TABLE old_rows (value TEXT)") + db.execute("INSERT INTO old_rows VALUES ('retained')") + migration = (ROOT / "intelligence-migrations/0038_payment_billing.sql").read_text() + db.executescript(migration) + db.executescript(migration) + assert db.execute("SELECT value FROM old_rows").fetchone()[0] == "retained" + store = pb.SqlPaymentStore(path) + calls, credited = [], [] + def http(url, *, headers, body): + calls.append((url, headers, parse_qs(body))) + fields = calls[-1][2] + return {"id": "cs_" + fields["client_reference_id"][0], "object": "checkout.session", + "url": "https://checkout.stripe.com/c/pay_test", "amount_total": int(fields["line_items[0][price_data][unit_amount]"][0]), + "currency": "usd", "livemode": False} + def callback(event): + credited.append(event) + return AuthorityResult(event.context, event.scoped_id, event.revision + 1, event.idempotency_id, True) + service = pb.PaymentBilling(store=store, env=ENV, http=http, secret_resolver=lambda ref: SECRET, + callback=callback, clock=lambda: NOW) + return SimpleNamespace(store=store, service=service, calls=calls, credited=credited, path=path) + + +def checkout(setup, **changes): + return setup.service.checkout(TenantContext("alice"), "alice", {**BODY, **changes}) + + +def event(row, kind="checkout.session.completed", **changes): + obj = {"id": "cs_" + row["checkout_id"], "client_reference_id": row["checkout_id"], + "metadata": {"checkout_id": row["checkout_id"]}, "payment_status": "paid", + "amount_total": row["amount_microusd"] // 10000, "currency": "usd", "livemode": False, + "payment_intent": "pi_test"} + obj.update(changes.pop("object_changes", {})) + return {"id": "evt_paid", "object": "event", "type": kind, "livemode": False, + "data": {"object": obj}, **changes} + + +def deliver(service, value): + raw = json.dumps(value, separators=(",", ":")).encode() + signature = hmac.new(SECRET.encode(), str(NOW).encode() + b"." + raw, hashlib.sha256).hexdigest() + return service.webhook(raw, f"t={NOW},v1={signature}") + + +@pytest.mark.parametrize("flag", [None, "", "false", "0", "junk"]) +def test_default_off_has_no_io(flag, caplog): + env = {**ENV, "PAYMENTS_ENABLED": flag} + service = pb.PaymentBilling(env=env, secret_resolver=lambda _: pytest.fail("disabled secret")) + with pytest.raises(pb.PaymentError) as caught: + service.checkout(TenantContext("alice"), "alice", BODY) + assert caught.value.status == 404 + with pytest.raises(pb.PaymentError): + service.webhook(VECTOR, "") + + +@pytest.mark.parametrize("config", ["{", "[]", '{"processor":"other"}', json.dumps({**CONFIG, "secret_key_ref": "bad ref"}), + json.dumps({**CONFIG, "return_origins": ["https://portal.test/path"]})]) +def test_invalid_configuration_disables_without_values(config, caplog): + pb._warn_once.cache_clear() + for _ in range(2): + assert pb.settings({**ENV, "PAYMENT_PROCESSOR_CONFIG_JSON": config}) is None + assert len(caplog.records) == 1 and config not in caplog.text + + +@pytest.mark.parametrize("amount", [True, 1, 499999, 500001, 1000000001, 500000.0, "500000"]) +def test_amount_contract_before_http(setup, amount): + with pytest.raises(pb.PaymentError) as caught: + checkout(setup, amount_microusd=amount) + assert caught.value.status == 400 and not setup.calls + + +@pytest.mark.parametrize("url", ["https://portal.test.evil/", "https://portal.test@evil.test/", "http://portal.test/", "https://portal.test:444/", "https://portal.test/\n"]) +def test_exact_return_origin(setup, url): + with pytest.raises(pb.PaymentError): + checkout(setup, return_url=url) + assert not setup.calls + + +def test_checkout_fixed_schema_idempotency_velocity_and_concurrency(setup): + with ThreadPoolExecutor(max_workers=4) as pool: + rows = list(pool.map(lambda _: checkout(setup), range(4))) + assert rows == [rows[0]] * 4 and len(setup.calls) == 1 + assert rows[0]["status"] == "pending" and not setup.credited + url, headers, fields = setup.calls[0] + assert url == "https://api.stripe.com/v1/checkout/sessions" + assert headers["Idempotency-Key"] == rows[0]["checkout_id"] + assert fields["metadata[checkout_id]"] == [rows[0]["checkout_id"]] + assert fields["client_reference_id"] == [rows[0]["checkout_id"]] + assert fields["line_items[0][price_data][unit_amount]"] == ["50"] + with pytest.raises(pb.PaymentError) as conflict: + checkout(setup, amount_microusd=510000) + assert conflict.value.status == 409 + for n in range(9): + checkout(setup, idempotency_key=f"order-{n+2}") + with pytest.raises(pb.PaymentError) as limited: + checkout(setup, idempotency_key="eleven") + assert limited.value.code == "payment_velocity_limited" and limited.value.status == 429 + + +def test_foreign_workspace_requires_explicit_permission(setup): + with pytest.raises(pb.PaymentError) as caught: + setup.service.checkout(TenantContext("alice", "foreign"), "alice", BODY) + assert caught.value.status == 403 and not setup.calls + setup.service.permission = lambda context, owner: context.org_id == "allowed" and owner == "alice" + assert setup.service.checkout(TenantContext("alice", "allowed"), "alice", BODY)["status"] == "pending" + + +def test_signature_shared_vector_rotation_raw_bytes_and_time(): + assert pb.verify_signature(VECTOR, f"t={NOW},v1={'0'*64},v1={SIGNATURE}", SECRET, NOW) + assert pb.verify_signature(VECTOR, f"t={NOW},v1={SIGNATURE}", SECRET, NOW + 300) + for raw, header, now in [(VECTOR+b" ", f"t={NOW},v1={SIGNATURE}", NOW), + (VECTOR, f"t={NOW},v0={SIGNATURE}", NOW), + (VECTOR, f"t={NOW},t={NOW},v1={SIGNATURE}", NOW), + (VECTOR, f"t={NOW},v1={SIGNATURE}", NOW + 301), + (VECTOR, f"t={NOW},v1={SIGNATURE}", NOW - 301)]: + assert not pb.verify_signature(raw, header, SECRET, now) + + +def test_invalid_signature_has_no_state_change(setup): + with pytest.raises(pb.PaymentError) as caught: + setup.service.webhook(VECTOR, f"t={NOW},v1={'0'*64}") + assert caught.value.status == 400 + with sqlite3.connect(setup.path) as db: + assert db.execute("SELECT COUNT(*) FROM payment_events").fetchone()[0] == 0 + + +def test_verified_payment_once_across_replays_and_success_types(setup): + row = checkout(setup) + value = event(row) + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(lambda _: deliver(setup.service, value), range(4))) + assert all(result["received"] for result in results) + assert len(setup.credited) == 1 + assert deliver(setup.service, event(row, "checkout.session.async_payment_succeeded", id="evt_async"))["received"] + assert len(setup.credited) == 1 and setup.credited[0].verified + assert setup.credited[0].amount == 500000 + with pytest.raises(Exception): + setup.credited[0].amount = 0 + + +@pytest.mark.parametrize("change", [{"account": "foreign"}, {"livemode": True}, + {"object_changes": {"currency": "eur"}}, + {"object_changes": {"amount_total": 51}}, + {"object_changes": {"id": "cs_other"}}]) +def test_mismatch_recorded_without_credit(setup, change): + row = checkout(setup) + assert deliver(setup.service, event(row, **change))["status"] == "mismatch" + assert not setup.credited + + +def test_delayed_payment_failure_unknown_and_missing_callback_retry(setup): + row = checkout(setup) + assert deliver(setup.service, event(row, object_changes={"payment_status": "unpaid"}))["status"] == "ignored" + assert not setup.credited + assert deliver(setup.service, event(row, "checkout.session.async_payment_failed", id="evt_failed"))["status"] == "failed" + assert deliver(setup.service, {"id": "evt_unknown", "type": "unknown", "object": "event"})["status"] == "ignored" + setup.service.callback = None + paid = event(row, "checkout.session.async_payment_succeeded", id="evt_retry") + with pytest.raises(pb.PaymentError) as unavailable: + deliver(setup.service, paid) + assert unavailable.value.code == "payment_credit_unavailable" + with sqlite3.connect(setup.path) as db: + assert db.execute("SELECT status FROM payment_events WHERE event_id='evt_retry'").fetchone()[0] == "pending_credit" + setup.service.callback = lambda e: AuthorityResult(e.context, e.scoped_id, e.revision + 1, e.idempotency_id, True) + assert deliver(setup.service, paid)["status"] == "credited" + + +def test_refund_cumulative_delta_and_dispute_append(setup): + row = checkout(setup) + deliver(setup.service, event(row)) + charge = {"id": "ch_test", "payment_intent": "pi_test", "amount": 50, "amount_refunded": 20, "currency": "usd", "livemode": False} + value = {"id": "evt_refund", "object": "event", "type": "charge.refunded", "livemode": False, "data": {"object": charge}} + assert deliver(setup.service, value)["status"] == "refunded" + deliver(setup.service, {**value, "id": "evt_refund_duplicate"}) + charge = {**charge, "amount_refunded": 50} + deliver(setup.service, {**value, "id": "evt_refund_rest", "data": {"object": charge}}) + dispute = {"id": "dp_test", "charge": "ch_test", "payment_intent": "pi_test", "amount": 50, "currency": "usd", "livemode": False} + deliver(setup.service, {**value, "id": "evt_dispute", "type": "charge.dispute.created", "data": {"object": dispute}}) + deliver(setup.service, {**value, "id": "evt_dispute_repeated", "type": "charge.dispute.created", "data": {"object": dispute}}) + assert [(e.kind, e.amount) for e in setup.credited] == [("credit", 500000), ("refund", 200000), ("refund", 300000), ("dispute", 500000)] + + +def test_callback_failure_keeps_stable_operation_and_no_false_success(setup): + row = checkout(setup) + operations = [] + def failure(e): + operations.append(e.idempotency_id) + raise RuntimeError("private detail") + setup.service.callback = failure + with pytest.raises(pb.PaymentError) as caught: + deliver(setup.service, event(row)) + assert caught.value.code == "payment_credit_unavailable" and "private" not in str(caught.value) + def success(e): + operations.append(e.idempotency_id) + return AuthorityResult(e.context, e.scoped_id, e.revision+1, e.idempotency_id, True) + setup.service.callback = success + deliver(setup.service, event(row)) + assert operations[0] == operations[1] + + +def test_refund_before_session_is_retried_and_reconciled(setup): + row = checkout(setup) + value = {"id": "evt_early", "object": "event", "type": "charge.refunded", "livemode": False, + "data": {"object": {"id": "ch_test", "payment_intent": "pi_test", "amount": 50, + "amount_refunded": 20, "currency": "usd", "livemode": False}}} + for _ in range(2): + with pytest.raises(pb.PaymentError) as caught: + deliver(setup.service, value) + assert caught.value.code == "payment_credit_unavailable" + assert not setup.credited + deliver(setup.service, event(row)) + assert deliver(setup.service, value)["status"] == "refunded" + assert [(e.kind, e.amount) for e in setup.credited] == [("credit", 500000), ("refund", 200000)] + with sqlite3.connect(setup.path) as db: + assert db.execute("SELECT COUNT(*) FROM payment_events WHERE event_id='evt_early'").fetchone()[0] == 1 + assert db.execute("SELECT COUNT(*) FROM payment_audit WHERE event_id='evt_early'").fetchone()[0] >= 3 + + +def test_missing_schema_fails_before_provider(setup, tmp_path): + setup.service.store = pb.SqlPaymentStore(tmp_path / "missing.sqlite") + with pytest.raises(pb.PaymentError) as caught: + checkout(setup) + assert caught.value.status == 503 and caught.value.code == "payments_unavailable" and not setup.calls + assert not (tmp_path / "missing.sqlite").exists() + + +def test_separate_instances_share_claims_and_callbacks(setup): + row = checkout(setup) + services = [pb.PaymentBilling(store=setup.store, env=ENV, secret_resolver=lambda _: SECRET, + callback=setup.service.callback, clock=lambda: NOW) for _ in range(4)] + def run(service): + try: + return deliver(service, event(row)) + except pb.PaymentError as error: + assert error.code == "payment_credit_unavailable" and error.status == 503 + return None + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(run, services)) + assert len(setup.credited) == 1 + assert deliver(services[0], event(row))["status"] == "credited" + + +def test_processor_failure_retries_same_opaque_id_without_new_velocity(setup): + original = setup.service.http + attempted = [] + def failure(url, *, headers, body): + attempted.append(headers["Idempotency-Key"]) + raise RuntimeError("private processor detail") + setup.service.http = failure + with pytest.raises(pb.PaymentError) as caught: + checkout(setup) + assert caught.value.code == "payment_processor_unavailable" + setup.service.http = original + row = checkout(setup) + assert attempted == [row["checkout_id"]] + with sqlite3.connect(setup.path) as db: + assert db.execute("SELECT attempts FROM payment_velocity").fetchone()[0] == 1 + + +def test_fixed_http_disallows_redirects_and_suppresses_processor_errors(monkeypatch): + calls = [] + def post(url, **kwargs): + calls.append((url, kwargs)) + return SimpleNamespace(status_code=302, json=lambda: pytest.fail("redirect body")) + monkeypatch.setattr(pb.requests, "post", post) + with pytest.raises(pb.PaymentError): + pb.stripe_http(pb.ENDPOINT, headers={}, body="mode=payment") + assert calls[0][1]["allow_redirects"] is False and calls[0][1]["timeout"] == (3, 10) + with pytest.raises(pb.PaymentError): + pb.stripe_http("https://foreign.test", headers={}, body="") + assert len(calls) == 1 + + +def test_d1_adapter_preserves_fixed_protocol_without_fallback(): + bodies = [] + def transport(body): + bodies.append(body) + return {"version": 1, "result": None} + store = pb.D1PaymentStore(transport) + assert store.call("find", checkout_id="pay_one", payment_intent=None) is None + assert bodies == [{"version": 1, "operation": "find", "checkout_id": "pay_one", "payment_intent": None}] + with pytest.raises(pb.PaymentError): + pb.D1PaymentStore(lambda _: {"version": 1}).call("find") + + +def test_missing_columns_and_webhook_schema_are_503(setup): + row = checkout(setup) + with sqlite3.connect(setup.path) as db: + db.execute("ALTER TABLE payment_events RENAME TO old_payment_events") + db.execute("CREATE TABLE payment_events (event_id TEXT)") + with pytest.raises(pb.PaymentError) as caught: + deliver(setup.service, event(row)) + assert caught.value.code == "payments_unavailable" and not setup.credited + + +@pytest.mark.parametrize("obj", [{"type": []}, {"type": "checkout.session.completed", "data": []}, + {"type": "checkout.session.completed", "data": {"object": []}}]) +def test_malformed_payload_never_crashes_or_credits(setup, obj): + result = deliver(setup.service, {"id": "evt_malformed", "object": "event", **obj}) + assert result["status"] in {"ignored", "mismatch"} and not setup.credited + + +def test_velocity_utc_reset_and_owner_isolation(setup): + for n in range(10): + checkout(setup, idempotency_key=f"day-{n}") + setup.service.clock = lambda: NOW+86400 + assert checkout(setup, idempotency_key="next-day")["status"] == "pending" + setup.service.checkout(TenantContext("bob"), "bob", BODY) + + +def test_permission_callback_failures_do_not_grant_access(setup): + setup.service.permission = lambda *_: "billing" + with pytest.raises(pb.PaymentError) as caught: + checkout(setup) + assert caught.value.status == 403 + def unavailable(*_): + raise RuntimeError("private permission detail") + setup.service.permission = unavailable + with pytest.raises(pb.PaymentError) as caught: + checkout(setup) + assert caught.value.status == 503 and not setup.calls + + +def test_registered_paths_off_auth_raw_webhook_redirect(setup, monkeypatch): + def authenticate(**kwargs): + def decorate(target): + def wrapper(*args, **kw): + if not g.get("authenticated_user"): + return {"error": "authentication_required"}, 401 + return target(*args, **kw) + wrapper.__name__ = target.__name__ + return wrapper + return decorate + monkeypatch.setattr(payments, "api_authenticate_only", authenticate) + app = Flask(__name__) + @app.before_request + def user(): + from flask import request + if request.headers.get("Authorization") == "Bearer test": + g.authenticated_user = {"id": "alice"} + exempted = [] + csrf = SimpleNamespace(exempt=lambda f: exempted.append(f.__name__) or f) + payments.register_payment_routes(app, csrf, service=setup.service) + client = app.test_client() + setup.service.env = {} + assert client.post("/v1/payments/checkout").status_code == 404 + assert client.post("/v1/payments/webhook").status_code == 404 + setup.service.env = ENV + assert client.post("/v1/payments/checkout", json=BODY).status_code == 401 + row = client.post("/v1/payments/checkout", json=BODY, headers={"Authorization": "Bearer test"}).json + assert client.get("/v1/payments/checkout").status_code == 405 + assert client.get("/paid").status_code == 404 and not setup.credited + assert client.post("/v1/payments/checkout", data="{", content_type="application/json", + headers={"Authorization": "Bearer test"}).status_code == 400 + assert client.post("/v1/payments/checkout", data="x"*(pb.MAX_BODY+1), content_type="application/json", + headers={"Authorization": "Bearer test"}).status_code == 400 + assert "payment_webhook" in exempted + raw = json.dumps(event(row)).encode() + signature = hmac.new(SECRET.encode(), str(NOW).encode()+b"."+raw, hashlib.sha256).hexdigest() + assert client.post("/v1/payments/webhook", data=raw, headers={"Stripe-Signature": f"t={NOW},v1={signature}"}).status_code == 200 + assert len(setup.credited) == 1 diff --git a/tests/test_payment_billing_worker.mjs b/tests/test_payment_billing_worker.mjs new file mode 100644 index 00000000..42ac8a1c --- /dev/null +++ b/tests/test_payment_billing_worker.mjs @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { createHmac } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { convertV4MiniflareOptions, Miniflare } from "miniflare"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; +import { handlePaymentStoreRequest, verifyPaymentSignature, paymentSettings } from "../worker/payment-webhooks.mjs"; + +const NOW = 1700000000; +const SECRET = "public-test-webhook-secret"; +const VECTOR = '{"id":"evt_vector","object":"event","type":"unknown","livemode":false}'; +const SIGNATURE = createHmac("sha256", SECRET).update(`${NOW}.${VECTOR}`).digest("hex"); +const ENV = { PAYMENTS_ENABLED: "true", PAYMENT_PROCESSOR_CONFIG_JSON: JSON.stringify({ processor: "stripe", + secret_key_ref: "PAYMENT_TEST_KEY", return_origins: ["https://portal.test"], product_name: "Gateway credits" }), + PAYMENT_WEBHOOK_KEY_REF: "PAYMENT_TEST_WEBHOOK" }; +const context = { principal_id: "alice", org_id: null, team_id: null, grants_revision: 0 }; + +function database(t, schema = true) { + const sql = new DatabaseSync(":memory:"); + t.after(() => sql.close()); + sql.exec("CREATE TABLE old_rows (value TEXT); INSERT INTO old_rows VALUES ('retained')"); + const migration = readFileSync(new URL("../intelligence-migrations/0038_payment_billing.sql", import.meta.url), "utf8"); + if (schema) { sql.exec(migration); sql.exec(migration); } + const db = { prepare(query) { + let values = []; + const statement = { bind(...args) { values = args; return statement; }, + first() { return sql.prepare(query).get(...values) ?? null; }, + all() { + const prepared = sql.prepare(query); + return prepared.columns().length ? { results: prepared.all(...values), meta: { changes: 0 } } + : { results: [], meta: { changes: Number(prepared.run(...values).changes) } }; + }, run() { return statement.all(); } }; + return statement; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { const result = statements.map(statement => statement.all()); sql.exec("COMMIT"); return result; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const env = { ...ENV, INTELLIGENCE_DB: db }; + const call = async (operation, data = {}) => { + const response = await handlePaymentStoreRequest(new Request("http://intelligence.internal/v1/managed-state/payments", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, operation, ...data }) }), env); + return { status: response.status, body: await response.json() }; + }; + return { sql, env, call }; +} +function reserve(overrides = {}) { + return { context, body_hash: "a".repeat(64), idempotency_key: "order-1", checkout_id: "pay_one", + amount: 500000, now: NOW, token: "token_one", ...overrides }; +} +function claim(overrides = {}) { + return { event_id: "evt_one", evidence_hash: "b".repeat(64), checkout_id: "pay_one", kind: "credit", + status: "processing", amount: 500000, target_amount: 0, operation_id: "credit:pay_one", + payment_intent: "pi_one", now: NOW, token: "claim_one", ...overrides }; +} +async function saved(call) { + assert.equal((await call("reserve", reserve())).status, 200); + assert.equal((await call("save", { checkout_id: "pay_one", token: "token_one", session_id: "cs_one", + url: "https://checkout.stripe.com/c/pay_test", livemode: false, merchant: "direct" })).status, 200); +} + +test("shared raw signature vectors, rotation, timestamp, no downgrade", async () => { + const raw = new TextEncoder().encode(VECTOR); + assert.equal(await verifyPaymentSignature(raw, `t=${NOW},v1=${"0".repeat(64)},v1=${SIGNATURE}`, SECRET, NOW), true); + assert.equal(await verifyPaymentSignature(raw, `t=${NOW},v1=${SIGNATURE}`, SECRET, NOW+300), true); + for (const [body, header, now] of [ + [VECTOR+" ", `t=${NOW},v1=${SIGNATURE}`, NOW], [VECTOR, `t=${NOW},v0=${SIGNATURE}`, NOW], + [VECTOR, `t=${NOW},t=${NOW},v1=${SIGNATURE}`, NOW], [VECTOR, `t=${NOW},v1=${SIGNATURE}`, NOW+301], + [VECTOR, `t=${NOW},v1=${SIGNATURE}`, NOW-301], [VECTOR, "garbage", NOW], + ]) assert.equal(await verifyPaymentSignature(new TextEncoder().encode(body), header, SECRET, now), false); +}); + +test("default off and invalid config perform no storage, warning excludes values", async () => { + for (const flag of [undefined, "", "false", "junk"]) { + const response = await handlePaymentStoreRequest(new Request("http://intelligence.internal/v1/managed-state/payments", { method: "POST" }), + { ...ENV, PAYMENTS_ENABLED: flag, INTELLIGENCE_DB: { prepare() { assert.fail("disabled storage"); } } }); + assert.equal(response.status, 404); + } + assert.equal(paymentSettings({ ...ENV, PAYMENT_PROCESSOR_CONFIG_JSON: "[]" }), null); + assert.equal(paymentSettings({ ...ENV, PAYMENT_WEBHOOK_KEY_REF: "bad ref" }), null); +}); + +test("private target validation and missing table fail closed", async t => { + const { env, call } = database(t, false); + assert.equal((await call("reserve", reserve())).status, 503); + for (const url of ["https://outside.test/v1/managed-state/payments", "http://intelligence.internal/v1/managed-state/payments?x=1"]) { + assert.equal((await handlePaymentStoreRequest(new Request(url, { method: "POST" }), env)).status, 404); + } + assert.equal((await call("unknown")).status, 400); + assert.equal((await call("reserve", reserve({ amount: 500001 }))).status, 400); + assert.equal((await call("reserve", { ...reserve(), secret: "unwanted" })).status, 400); + assert.equal((await call("reserve", reserve({ idempotency_key: "order-1\n" }))).status, 400); +}); + +test("migration preserves old rows, reserve CAS scopes, velocity and retry", async t => { + const { sql, call } = database(t); + assert.equal(sql.prepare("SELECT value FROM old_rows").get().value, "retained"); + const rows = await Promise.all(Array.from({ length: 4 }, () => call("reserve", reserve()))); + assert.ok(rows.every(row => row.status === 200 && row.body.result.checkout_id === "pay_one")); + assert.equal(sql.prepare("SELECT attempts FROM payment_velocity").get().attempts, 1); + assert.equal((await call("reserve", reserve({ body_hash: "c".repeat(64) }))).status, 409); + for (let n = 2; n <= 10; n++) assert.equal((await call("reserve", reserve({ idempotency_key: `order-${n}`, checkout_id: `pay_${n}` }))).status, 200); + assert.equal((await call("reserve", reserve({ idempotency_key: "eleven", checkout_id: "pay_11" }))).status, 429); + assert.equal((await call("reserve", reserve({ context: { ...context, principal_id: "bob" }, checkout_id: "pay_bob" }))).status, 200); + assert.equal((await call("reserve", reserve({ context: { ...context, org_id: "org_one" }, checkout_id: "pay_org", now: NOW+86400 }))).status, 200); +}); + +test("event CAS before callback, retries and distinct completion events credit once", async t => { + const { call, sql } = database(t); + await saved(call); + const rows = await Promise.all(Array.from({ length: 4 }, (_, n) => call("claim", claim({ token: `claim_${n}` })))); + assert.equal(rows.filter(row => row.status === 200 && row.body.result.claimed).length, 1); + const winner = rows.find(row => row.status === 200).body.result; + assert.equal((await call("finish", { event_id: "evt_one", token: winner.claim_token, status: "pending_credit", now: NOW })).status, 200); + const retry = await call("claim", claim({ token: "claim_retry" })); + assert.equal(retry.body.result.operation_id, "credit:pay_one"); + assert.equal((await call("finish", { event_id: "evt_one", token: "claim_retry", status: "credited", now: NOW })).status, 200); + const duplicate = await call("claim", claim()); + assert.equal(duplicate.body.result.claimed, false); + const asyncEvent = await call("claim", claim({ event_id: "evt_async" })); + assert.equal(asyncEvent.body.result.claimed, false); + assert.equal(sql.prepare("SELECT credited,revision FROM payment_checkouts").get().credited, 1); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM payment_events").get().n, 2); +}); + +test("refund deltas, operation deduplication and content-free audit", async t => { + const { call, sql } = database(t); + await saved(call); + await call("claim", claim()); + await call("finish", { event_id: "evt_one", token: "claim_one", status: "credited", now: NOW }); + const refund = claim({ event_id: "evt_refund", kind: "refund", target_amount: 200000, + operation_id: "refund:pay_one:200000", token: "refund_one" }); + assert.equal((await call("claim", refund)).body.result.amount, 200000); + await call("finish", { event_id: "evt_refund", token: "refund_one", status: "refunded", now: NOW }); + assert.equal((await call("claim", { ...refund, event_id: "evt_refund_again" })).body.result.claimed, false); + assert.equal(sql.prepare("SELECT refunded FROM payment_checkouts").get().refunded, 200000); + assert.deepEqual(sql.prepare("PRAGMA table_info(payment_audit)").all().map(row => row.name), ["id", "checkout_id", "event_id", "outcome", "created_at"]); + assert.equal((await call("finish", { event_id: "evt_refund", token: "foreign", status: "refunded", now: NOW })).status, 503); +}); + +test("unmatched event evidence resumes after the verified intent is linked", async t => { + const { call, sql } = database(t); + await saved(call); + const unmatched = claim({ event_id: "evt_early", checkout_id: null, kind: "ignored", status: "pending_match", + amount: 0, target_amount: 0, operation_id: null, payment_intent: null }); + assert.equal((await call("claim", unmatched)).body.result.status, "pending_match"); + await call("claim", claim()); + await call("finish", { event_id: "evt_one", token: "claim_one", status: "credited", now: NOW }); + const matched = claim({ event_id: "evt_early", kind: "refund", target_amount: 200000, + operation_id: "refund:pay_one:200000", token: "refund_early" }); + assert.equal((await call("claim", matched)).body.result.amount, 200000); + await call("finish", { event_id: "evt_early", token: "refund_early", status: "refunded", now: NOW }); + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM payment_events WHERE event_id='evt_early'").get().n, 1); +}); + +test("unknown events and mismatches are recorded without credit claims", async t => { + const { call, sql } = database(t); + for (const status of ["ignored", "mismatch"]) { + const result = await call("claim", claim({ event_id: `evt_${status}`, checkout_id: null, + kind: "ignored", status, amount: 0, operation_id: null, payment_intent: null })); + assert.equal(result.status, 200); + assert.equal(result.body.result.claimed, false); + } + assert.equal(sql.prepare("SELECT COUNT(*) AS n FROM payment_events").get().n, 2); +}); + +test("Worker forwards unauthenticated webhook raw bytes and signature to the Container", async t => { + const worker = (await loadWorkerModule()).default; + const raw = new TextEncoder().encode(' {"fixture":"é"}\r\n'); + let forwards = 0; + t.mock.method(globalThis, "fetch", () => assert.fail("unexpected external network")); + const env = { MULTILLM_PROXY_CONTAINER: { getByName(name) { + assert.equal(name, "primary"); + return { async fetch(request) { + forwards++; + assert.equal(new URL(request.url).pathname, "/v1/payments/webhook"); + assert.equal(request.headers.get("Authorization"), null); + assert.equal(request.headers.get("Stripe-Signature"), "signature-fixture"); + assert.deepEqual(new Uint8Array(await request.arrayBuffer()), raw); + return Response.json({ received: true }); + } }; + } } }; + const response = await worker.fetch(new Request("https://gateway.test/v1/payments/webhook", { + method: "POST", headers: { "content-type": "application/json", "Stripe-Signature": "signature-fixture" }, body: raw }), env, + { waitUntil() {} }); + assert.equal(response.status, 200); + assert.equal(forwards, 1); +}); + +test("real D1 batches preserve velocity and event CAS", async t => { + const entry = fileURLToPath(new URL("../worker/payment-test-entry.mjs", import.meta.url)); + const modules = [ + { type: "ESModule", path: entry, contents: 'import {handlePaymentStoreRequest} from "./payment-webhooks.mjs"; export default {fetch(request,env) {return handlePaymentStoreRequest(new Request("http://intelligence.internal/v1/managed-state/payments",request),env);}}' }, + ...["payment-webhooks.mjs", "enterprise-contract.mjs"].map(name => ({ type: "ESModule", + path: fileURLToPath(new URL(`../worker/${name}`, import.meta.url)), + contents: readFileSync(new URL(`../worker/${name}`, import.meta.url), "utf8") })), + ]; + const mf = new Miniflare(convertV4MiniflareOptions({ modules, compatibilityDate: "2026-08-18", + d1Databases: ["INTELLIGENCE_DB"], bindings: ENV })); + t.after(() => mf.dispose()); + const db = await mf.getD1Database("INTELLIGENCE_DB"); + const migration = readFileSync(new URL("../intelligence-migrations/0038_payment_billing.sql", import.meta.url), "utf8"); + await db.batch(migration.split(";").map(sql => sql.trim()).filter(Boolean).map(sql => db.prepare(sql))); + const call = async (operation, data) => { + const response = await mf.dispatchFetch("https://payment.test/", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ version: 1, operation, ...data }) }); + return { status: response.status, body: await response.json() }; + }; + await saved(call); + assert.equal((await call("reserve", reserve())).body.result.checkout_id, "pay_one"); + assert.equal((await db.prepare("SELECT attempts FROM payment_velocity").first()).attempts, 1); + const claims = await Promise.all(Array.from({ length: 4 }, (_, n) => call("claim", claim({ token: `claim_${n}` })))); + assert.equal(claims.filter(result => result.status === 200 && result.body.result.claimed).length, 1); + const token = claims.find(result => result.status === 200).body.result.claim_token; + assert.equal((await call("finish", { event_id: "evt_one", token, status: "credited", now: NOW })).status, 200); + assert.equal((await call("claim", claim())).body.result.claimed, false); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..31da6d9b 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,9 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "PAYMENTS_ENABLED", + "PAYMENT_PROCESSOR_CONFIG_JSON", + "PAYMENT_WEBHOOK_KEY_REF", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..83f2cb51 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "payment_checkouts", "payment_events", "payment_velocity", "payment_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/payment-webhooks.mjs b/worker/payment-webhooks.mjs new file mode 100644 index 00000000..880d19b0 --- /dev/null +++ b/worker/payment-webhooks.mjs @@ -0,0 +1,250 @@ +/** Private, durable payment domain. Public Stripe delivery stays in the Container. */ +import { TenantContext } from "./enterprise-contract.mjs"; + +const MAX_BODY = 65536, LEASE = 60; +const LABEL = /^[A-Za-z0-9_:.-]{1,128}(?![\s\S])/, REF = /^[A-Za-z_][A-Za-z0-9_]{0,127}(?![\s\S])/, HASH = /^[0-9a-f]{64}(?![\s\S])/; +const columns = { + payment_checkouts: "checkout_id,principal,context_json,idempotency_key,body_hash,amount_microusd,currency,status,checkout_url,session_id,payment_intent,merchant,livemode,credited,refunded,revision,claim_token,claim_until,active_event,created_at", + payment_events: "event_id,evidence_hash,checkout_id,kind,status,amount,target_amount,operation_id,revision,claim_token,claim_until,created_at", + payment_velocity: "principal,utc_day,attempts", payment_audit: "id,checkout_id,event_id,outcome,created_at", +}; +let warnedFlag = false, warnedConfig = false; +class PaymentError extends Error { + constructor(code = "payments_unavailable", status = 503) { super(code); this.code = code; this.status = status; } +} +function invalid() { throw new PaymentError("invalid_payment_operation", 400); } +function label(value, nullable = false) { + if (nullable && value === null) return; + if (typeof value !== "string" || !LABEL.test(value)) invalid(); +} +function integer(value, maximum = 1000000000) { + if (!Number.isSafeInteger(value) || value < 0 || value > maximum) invalid(); +} +function origin(value) { + if (typeof value !== "string" || value.length > 2048 || /[\x00-\x20\x7f\\]/.test(value)) invalid(); + const url = new URL(value); + if (url.protocol !== "https:" || !url.hostname || url.username || url.password) invalid(); + return url.origin; +} +export function paymentSettings(env = {}, warn = message => console.warn(message)) { + const raw = env.PAYMENTS_ENABLED; + const flag = raw == null ? "" : typeof raw === "string" ? raw.trim().toLowerCase() : "invalid"; + if (["", "0", "false", "off", "no"].includes(flag)) return null; + if (!["1", "true", "on", "yes"].includes(flag)) { + if (!warnedFlag) { warnedFlag = true; warn("Invalid payment flag; payments disabled"); } + return null; + } + try { + const config = JSON.parse(env.PAYMENT_PROCESSOR_CONFIG_JSON || "{}"); + if (!config || Array.isArray(config) || Object.keys(config).sort().join() !== "processor,product_name,return_origins,secret_key_ref" + || config.processor !== "stripe" || typeof config.secret_key_ref !== "string" || !REF.test(config.secret_key_ref) + || typeof env.PAYMENT_WEBHOOK_KEY_REF !== "string" || !REF.test(env.PAYMENT_WEBHOOK_KEY_REF) + || typeof config.product_name !== "string" || config.product_name.length < 1 || config.product_name.length > 128 + || /[\x00-\x1f]/.test(config.product_name) || !Array.isArray(config.return_origins) + || config.return_origins.length < 1 || config.return_origins.length > 16 + || config.return_origins.some(value => origin(value) !== value)) throw new Error(); + return Object.freeze({ ...config, webhook_key_ref: env.PAYMENT_WEBHOOK_KEY_REF }); + } catch { + if (!warnedConfig) { warnedConfig = true; warn("Invalid payment configuration; payments disabled"); } + return null; + } +} + +export async function verifyPaymentSignature(raw, header, secret, now) { + try { + if (!(raw instanceof Uint8Array) || raw.byteLength > MAX_BODY || typeof secret !== "string" || !secret + || typeof header !== "string" || header.length > 8192 || !Number.isSafeInteger(now)) return false; + const pieces = header.split(",").map(piece => { + const index = piece.indexOf("="); + if (index < 0) throw new Error(); + return [piece.slice(0, index).trim(), piece.slice(index+1).trim()]; + }); + const timestamps = pieces.filter(([key]) => key === "t").map(([, value]) => value); + if (timestamps.length !== 1 || !/^[0-9]{1,12}$/.test(timestamps[0]) || Math.abs(now-Number(timestamps[0])) > 300) return false; + const prefix = new TextEncoder().encode(`${timestamps[0]}.`); + const signed = new Uint8Array(prefix.length+raw.length); signed.set(prefix); signed.set(raw, prefix.length); + const key = await crypto.subtle.importKey("raw", new TextEncoder().encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["verify"]); + // WebCrypto performs the HMAC comparison rather than JavaScript string equality. + let matched = false; + for (const [scheme, value] of pieces) { + if (scheme !== "v1" || !HASH.test(value)) continue; + const signature = Uint8Array.from(value.match(/../g).map(byte => parseInt(byte, 16))); + matched = await crypto.subtle.verify("HMAC", key, signature, signed) || matched; + } + return matched; + } catch { return false; } +} + +function checked(body) { + const fields = { + reserve: ["context", "body_hash", "idempotency_key", "checkout_id", "amount", "now", "token"], + save: ["checkout_id", "token"], find: ["checkout_id", "payment_intent"], + claim: ["event_id", "evidence_hash", "checkout_id", "kind", "status", "amount", "target_amount", "operation_id", "payment_intent", "now", "token"], + finish: ["event_id", "token", "status", "now"], + }; + if (!body || Array.isArray(body) || body.version !== 1 || !Object.hasOwn(fields, body.operation)) invalid(); + const optional = body.operation === "save" ? ["session_id", "url", "livemode", "merchant"] : []; + const required = ["version", "operation", ...fields[body.operation]]; + if (required.some(name => !Object.hasOwn(body, name)) || Object.keys(body).some(name => !required.includes(name) && !optional.includes(name))) invalid(); + for (const name of ["checkout_id", "payment_intent", "operation_id"]) if (name in body) label(body[name], true); + for (const name of ["token", "event_id", "idempotency_key"]) if (name in body) label(body[name]); + for (const name of ["now", "amount", "target_amount"]) if (name in body) integer(body[name], name === "now" ? 999999999999 : 1000000000); + for (const name of ["body_hash", "evidence_hash"]) if (name in body && (typeof body[name] !== "string" || !HASH.test(body[name]))) invalid(); + if (body.operation === "reserve") { + body.context = new TenantContext(body.context); + if (body.amount < 500000 || body.amount % 10000 || !body.checkout_id) invalid(); + } + if (body.operation === "save" && body.session_id != null) { + label(body.session_id); + if (origin(body.url) !== "https://checkout.stripe.com" || typeof body.livemode !== "boolean" || body.merchant !== "direct" + || optional.some(name => !Object.hasOwn(body, name))) invalid(); + } else if (body.operation === "save" && optional.some(name => Object.hasOwn(body, name))) invalid(); + if (body.operation === "claim") { + if (!["credit", "refund", "dispute", "ignored"].includes(body.kind) || !["processing", "ignored", "mismatch", "failed", "pending_match"].includes(body.status)) invalid(); + if (body.status === "processing" && (!body.checkout_id || !body.operation_id || body.kind === "ignored" || body.amount === 0)) invalid(); + if (body.kind === "refund" && body.target_amount > body.amount) invalid(); + } + if (body.operation === "finish" && !["pending_credit", "credited", "refunded", "disputed"].includes(body.status)) invalid(); + return body; +} +async function ready(env) { + const db = env.INTELLIGENCE_DB; + if (!db?.prepare || !db?.batch) throw new PaymentError(); + for (const [table, names] of Object.entries(columns)) await db.prepare(`SELECT ${names} FROM ${table} LIMIT 0`).all(); + return db; +} +function canonicalContext(context) { + return JSON.stringify({ grants_revision: context.grants_revision, org_id: context.org_id, + principal_id: context.principal_id, team_id: context.team_id }); +} +const stmt = (db, sql, ...args) => db.prepare(sql).bind(...args); + +async function reserve(db, b) { + const scope = canonicalContext(b.context), principal = b.context.principal_id; + const lookup = () => stmt(db, "SELECT * FROM payment_checkouts WHERE principal=? AND context_json=? AND idempotency_key=?", principal, scope, b.idempotency_key).first(); + let row = await lookup(); + if (row && row.body_hash !== b.body_hash) throw new PaymentError("payment_idempotency_conflict", 409); + if (!row) { + const day = new Date(b.now*1000).toISOString().slice(0, 10); + await db.batch([ + stmt(db, "INSERT INTO payment_velocity VALUES (?,?,0) ON CONFLICT DO NOTHING", principal, day), + stmt(db, "INSERT INTO payment_checkouts (checkout_id,principal,context_json,idempotency_key,body_hash,amount_microusd,currency,claim_token,claim_until,created_at) SELECT ?,?,?,?,?,?,'USD',?,?,? WHERE EXISTS (SELECT 1 FROM payment_velocity WHERE principal=? AND utc_day=? AND attempts<10) ON CONFLICT(principal,context_json,idempotency_key) DO NOTHING", + b.checkout_id, principal, scope, b.idempotency_key, b.body_hash, b.amount, b.token, b.now+LEASE, b.now, principal, day), + stmt(db, "UPDATE payment_velocity SET attempts=attempts+1 WHERE principal=? AND utc_day=? AND changes()=1", principal, day), + stmt(db, "INSERT INTO payment_audit (checkout_id,outcome,created_at) SELECT ?,'created',? WHERE changes()=1", b.checkout_id, b.now), + ]); + row = await lookup(); + if (!row) throw new PaymentError("payment_velocity_limited", 429); + if (row.body_hash !== b.body_hash) throw new PaymentError("payment_idempotency_conflict", 409); + } else if (!row.checkout_url && row.claim_until <= b.now) { + await stmt(db, "UPDATE payment_checkouts SET claim_token=?,claim_until=? WHERE checkout_id=? AND claim_until<=? AND checkout_url IS NULL", + b.token, b.now+LEASE, row.checkout_id, b.now).run(); + row = await lookup(); + } + return row; +} +async function save(db, b) { + if (b.session_id == null) { + await stmt(db, "UPDATE payment_checkouts SET claim_until=0 WHERE checkout_id=? AND claim_token=?", b.checkout_id, b.token).run(); + } else { + const result = await stmt(db, "UPDATE payment_checkouts SET session_id=?,checkout_url=?,livemode=?,merchant=?,status='pending',claim_until=0,claim_token=NULL WHERE checkout_id=? AND claim_token=?", + b.session_id, b.url, Number(b.livemode), b.merchant, b.checkout_id, b.token).run(); + if (result.meta.changes !== 1) throw new PaymentError("payment_checkout_processing", 503); + } + return null; +} +async function find(db, b) { + return b.checkout_id ? await stmt(db, "SELECT * FROM payment_checkouts WHERE checkout_id=?", b.checkout_id).first() + : await stmt(db, "SELECT * FROM payment_checkouts WHERE payment_intent=?", b.payment_intent).first(); +} +async function claim(db, b) { + let existing = await stmt(db, "SELECT * FROM payment_events WHERE event_id=?", b.event_id).first(); + if (existing) { + if (existing.evidence_hash !== b.evidence_hash) throw new PaymentError("payment_event_conflict", 400); + if (!["processing", "pending_credit", "pending_match"].includes(existing.status)) return { ...existing, claimed: false }; + if (existing.status === "pending_match") { + if (b.status === "pending_match") return { ...existing, claimed: false }; + existing = null; + } + if (existing && existing.claim_until > b.now) throw new PaymentError("payment_credit_unavailable", 503); + } + const row = b.checkout_id ? await find(db, b) : null; + let status = b.status, amount = b.amount; + if (status === "processing") { + if (!row || (row.active_event && row.active_event !== b.event_id)) throw new PaymentError("payment_credit_unavailable", 503); + if (!existing) { + const prior = await stmt(db, "SELECT status FROM payment_events WHERE operation_id=? AND status IN ('credited','refunded','disputed') LIMIT 1", b.operation_id).first(); + if (prior || (b.kind === "credit" && row.credited)) status = "ignored"; + if (b.kind !== "credit" && !row.credited) throw new PaymentError("payment_credit_unavailable", 503); + if (b.kind === "refund") { amount = Math.max(0, b.target_amount-row.refunded); if (amount === 0) status = "ignored"; } + } + } + const statements = []; + if (existing) { + statements.push(stmt(db, "UPDATE payment_events SET status='processing',claim_token=?,claim_until=? WHERE event_id=? AND claim_until<=? AND EXISTS (SELECT 1 FROM payment_checkouts WHERE checkout_id=? AND active_event=? AND revision=?)", + b.token, b.now+LEASE, b.event_id, b.now, existing.checkout_id, b.event_id, existing.revision)); + } else { + const guard = row ? " WHERE EXISTS (SELECT 1 FROM payment_checkouts WHERE checkout_id=? AND revision=? AND (active_event IS NULL OR active_event=?))" : ""; + statements.push(stmt(db, "INSERT INTO payment_events (event_id,evidence_hash,checkout_id,kind,status,amount,target_amount,operation_id,revision,claim_token,claim_until,created_at) SELECT ?,?,?,?,?,?,?,?,?,?,?,?" + guard + " ON CONFLICT(event_id) DO UPDATE SET checkout_id=excluded.checkout_id,kind=excluded.kind,status=excluded.status,amount=excluded.amount,target_amount=excluded.target_amount,operation_id=excluded.operation_id,revision=excluded.revision,claim_token=excluded.claim_token,claim_until=excluded.claim_until WHERE payment_events.status='pending_match' AND payment_events.evidence_hash=excluded.evidence_hash", + b.event_id, b.evidence_hash, b.checkout_id, b.kind, status, amount, b.target_amount, b.operation_id, row?.revision ?? 0, + b.token, status === "processing" ? b.now+LEASE : 0, b.now, ...(row ? [row.checkout_id, row.revision, b.event_id] : []))); + } + if (status === "processing") { + statements.push(stmt(db, "UPDATE payment_checkouts SET active_event=?,payment_intent=COALESCE(payment_intent,?) WHERE checkout_id=? AND EXISTS (SELECT 1 FROM payment_events WHERE event_id=? AND claim_token=? AND status='processing')", + b.event_id, b.payment_intent, b.checkout_id, b.event_id, b.token)); + } else if (status === "failed" && row && !row.credited) { + statements.push(stmt(db, "UPDATE payment_checkouts SET status='failed' WHERE checkout_id=? AND credited=0 AND EXISTS (SELECT 1 FROM payment_events WHERE event_id=? AND claim_token=?)", row.checkout_id, b.event_id, b.token)); + } + statements.push(stmt(db, "INSERT INTO payment_audit (checkout_id,event_id,outcome,created_at) SELECT ?,?,?,? WHERE EXISTS (SELECT 1 FROM payment_events WHERE event_id=? AND claim_token=?)", + b.checkout_id, b.event_id, status, b.now, b.event_id, b.token)); + const results = await db.batch(statements); + existing = await stmt(db, "SELECT * FROM payment_events WHERE event_id=?", b.event_id).first(); + if (!existing || (results[0].meta.changes !== 1 && ["processing", "pending_credit"].includes(existing.status))) throw new PaymentError("payment_credit_unavailable", 503); + if (existing.evidence_hash !== b.evidence_hash) throw new PaymentError("payment_event_conflict", 400); + return { ...existing, claimed: existing.status === "processing" && existing.claim_token === b.token }; +} +async function finish(db, b) { + const row = await stmt(db, "SELECT * FROM payment_events WHERE event_id=? AND claim_token=? AND status='processing'", b.event_id, b.token).first(); + if (!row) throw new PaymentError("payment_credit_unavailable", 503); + const expected = { credit: "credited", refund: "refunded", dispute: "disputed" }[row.kind]; + if (b.status !== "pending_credit" && b.status !== expected) invalid(); + const statements = [stmt(db, "UPDATE payment_events SET status=?,claim_until=0 WHERE event_id=? AND claim_token=? AND status='processing'", b.status, b.event_id, b.token)]; + if (b.status !== "pending_credit") statements.push(stmt(db, "UPDATE payment_checkouts SET active_event=NULL,revision=revision+1,credited=CASE WHEN ?='credit' THEN 1 ELSE credited END,refunded=CASE WHEN ?='refund' THEN ? ELSE refunded END,status=? WHERE checkout_id=? AND active_event=? AND revision=? AND changes()=1", + row.kind, row.kind, row.target_amount, b.status, row.checkout_id, b.event_id, row.revision)); + statements.push(stmt(db, "INSERT INTO payment_audit (checkout_id,event_id,outcome,created_at) SELECT ?,?,?,? WHERE changes()=1", row.checkout_id, b.event_id, b.status, b.now)); + const results = await db.batch(statements); + if (results[0].meta.changes !== 1) throw new PaymentError("payment_credit_unavailable", 503); + return null; +} +const operations = { reserve, save, find, claim, finish }; + +export async function handlePaymentStoreRequest(request, env) { + const failed = error => Response.json({ error: { code: error instanceof PaymentError ? error.code : "payments_unavailable", + message: "Payment operation unavailable." } }, { status: error instanceof PaymentError ? error.status : 503, headers: { "Cache-Control": "no-store" } }); + if (!paymentSettings(env)) return failed(new PaymentError("not_found", 404)); + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/managed-state/payments" + || url.search || url.hash || url.username || url.password || request.method !== "POST") return failed(new PaymentError("not_found", 404)); + let body; + try { + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") invalid(); + const reader = request.body?.getReader(); + if (!reader) invalid(); + const chunks = []; let length = 0; + try { + while (true) { + const { done, value } = await reader.read(); if (done) break; + length += value.length; + if (length > MAX_BODY) { await reader.cancel(); invalid(); } + chunks.push(value); + } + } finally { reader.releaseLock(); } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } + body = checked(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + } catch { return failed(new PaymentError("invalid_payment_operation", 400)); } + try { + const db = await ready(env), result = await operations[body.operation](db, body); + return Response.json({ version: 1, result }, { headers: { "Cache-Control": "no-store" } }); + } catch (error) { return failed(error); } +} From 248addddf95aedb9eb5754af480d3b37d2070dd4 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:29:24 +0530 Subject: [PATCH 3/9] feat(provisioning): add opt-in SCIM 2.0 user and group provisioning SCIM_ENABLED (off by default) serves /scim/v2 discovery, Users and Groups with SCIM errors, ETag versions and If-Match checks, count-bounded pagination and a small documented filter subset. A provisioning bearer token is matched by digest against tenant-scoped secret references, and fixes the organisation. Provisioned accounts are never administrators and receive no usable key over SCIM. Deactivation or deletion revokes the account's key and sessions through the security revision and keeps the account, external identity and usage. Groups map to teams through the tenant authority. Storage is in D1 (migration 0036). --- docs/scim-provisioning.md | 113 +++++ .../0036_scim_provisioning.sql | 37 ++ package.json | 2 +- routes/scim.py | 145 ++++++ services/scim_provisioning.py | 444 ++++++++++++++++++ services/user_provisioning.py | 33 +- tests/fixtures/secret_firewall_coverage.json | 5 + tests/test_scim_provisioning.py | 376 +++++++++++++++ tests/test_scim_provisioning_worker.mjs | 208 ++++++++ worker/container-env.mjs | 2 + worker/d1-schema.mjs | 2 +- worker/scim-d1.mjs | 248 ++++++++++ 12 files changed, 1612 insertions(+), 3 deletions(-) create mode 100644 docs/scim-provisioning.md create mode 100644 intelligence-migrations/0036_scim_provisioning.sql create mode 100644 routes/scim.py create mode 100644 services/scim_provisioning.py create mode 100644 tests/test_scim_provisioning.py create mode 100644 tests/test_scim_provisioning_worker.mjs create mode 100644 worker/scim-d1.mjs diff --git a/docs/scim-provisioning.md b/docs/scim-provisioning.md new file mode 100644 index 00000000..f38af208 --- /dev/null +++ b/docs/scim-provisioning.md @@ -0,0 +1,113 @@ +# SCIM user and group provisioning + +SCIM 2.0 provisions non-administrator gateway accounts and organisation teams. The +API is disabled by default. Unset, empty, `false`, `0`, `off` and `no` values for +`SCIM_ENABLED` return 404 for every `/scim/v2/*` route. An invalid value also disables +the API and emits one warning without the value. Existing account creation is unchanged. + +## Configuration and authentication + +Set `SCIM_ENABLED=true` only after applying the SCIM storage migration and configuring +the account and tenant authorities. `AUTH_STORAGE_BACKEND` and +`INTELLIGENCE_STORAGE_BACKEND` must both be `d1`, and +`CONFIG_REVISION_SYNC_ENABLED=true` must be enabled for the Worker and Container. +Security revisions expire within `CONFIG_SECURITY_TTL_SECONDS` (default and maximum: +five seconds); an unavailable or stale security copy denies generation. + +Configure token references, never token values: + +```json +{"tenants":{"org:example":{"token_ref":"SCIM_PROVISIONING_EXAMPLE"}}} +``` + +This is the value of `SCIM_TRUST_CONFIG_JSON`. The named environment variable or +Worker secret binding supplies the provisioning token privately. Supply each reference +to the Container through the deployment's reviewed secret-binding configuration. +An empty trust config uses `{}` and authorises nobody. Each tenant needs a distinct +token. Missing, unknown, ambiguous or invalid trust entries return a SCIM 401 error. +The server compares SHA-256 digests in constant time, fixes the organisation from the +matched reference, and ignores caller-supplied organisation headers. Provisioning +tokens and account credentials are never returned or logged. + +Send the token in `Authorization: Bearer `. Browser sessions do +not authorise this API. It is exempt from dashboard CSRF checks and accepts +`application/scim+json` or `application/json` request bodies. Responses use +`application/scim+json` and `Cache-Control: no-store`. + +## Endpoints and supported attributes + +The discovery endpoints are `/scim/v2/ServiceProviderConfig`, `/scim/v2/Schemas` +and `/scim/v2/ResourceTypes`; schema and resource-type entries can also be read by ID. +`Users` and `Groups` support list, get, POST, PUT, PATCH and DELETE. PATCH uses the +SCIM PatchOp schema and `Operations` array. Unsupported schemas, attributes, paths, +operations and list parameters return a SCIM 400 error. + +Users accept `userName`, `externalId`, `active`, `displayName`, `name` and `emails`. +Names support `formatted`, `familyName`, `givenName`, `middleName`, `honorificPrefix` +and `honorificSuffix`. Emails support `value`, `type`, `display` and `primary`, with +at most one primary entry. `userName` is immutable and globally unique among gateway +accounts. IDs and external identities remain stable across deactivation. +Administrator roles, permission escalation and environment-managed administrator +names cannot be provisioned. New accounts receive only the default `chat` and +`models` scopes. Their generated key is discarded; an authorised operator must issue +a new key through the existing key-rotation flow when key access is needed. + +Groups accept `displayName`, `externalId` and `members` (`value`, optional `display` +and `$ref`). Member values are SCIM user IDs in the same organisation. Membership +must also pass the tenant authority. Teams and members cannot grant administrator +roles. Group writes require an atomic team authority; missing authorities return 503. + +PATCH supports `add`, `replace` and `remove` for mutable user attributes and group +attributes. A missing path with an object value applies its named mutable attributes. +Adding a name merges its subattributes; adding emails appends entries; adding members +merges by member ID. Removing `active` deactivates the account. A member can be removed +with `members[value eq ""]`. Other selectors and nested PATCH paths are unsupported. +PUT replaces the supported attributes; omitted user `active` defaults to true and +omitted group members defaults to an empty list. + +For example, create a user using the User schema: + +```json +{"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"userName":"alice","externalId":"directory-subject-1","active":true} +``` + +POST returns 201 with `Location` and `ETag`. Repeating an external ID in the same +organisation and resource type returns its existing resource with 200, including +after deactivation. A conflicting username returns 409 `uniqueness`. + +Each resource's `meta.version` is its weak ETag. Send the current ETag in `If-Match` +on PUT, PATCH or DELETE. A stale ETag returns 412; omitting the header still uses +an atomic stored-version check, so concurrent writers cannot silently overwrite. +`If-Match: *` requires the resource to exist. + +## Limits, deactivation and retention + +List responses use `startIndex` (one-based, default 1) and `count` (default 100, +maximum 100; zero returns no rows). The only filter expressions are +`userName eq "..."`, `externalId eq "..."` and `displayName eq "..."`. Other expressions +return 400 `invalidFilter`. Sorting, projections, bulk operations and password changes +are unsupported. Requests are limited to 128 KiB, 100 PATCH operations, 100 email +entries and 1,000 group members. Page positions are not snapshot cursors. + +Setting `active=false` or deleting a user deactivates it. The account, external identity +and historical usage remain. The server rotates and discards its key, marks it revoked, +and advances the existing key-control and model-grant security revisions in the same +transaction. Dashboard sessions fail persisted revocation and key-prefix checks. +Reactivation clears the revocation marker without restoring the old key or sessions. +DELETE returns 204. Deleting a group removes its current memberships and deactivates +its mapped team while retaining the group identity and audit history. + +SCIM storage contains directory identity attributes, external mappings, token digests, +membership state and content-free operation audits. It does not record prompts, +generated responses or provisioning bearer values. These records have no automatic +retention expiry; deactivation does not purge them. Operators must account for that +retention when enrolling a directory. SCIM has no provider calls, inference cost, +payment processing or automatic billing; D1 operations incur the deployment's normal +storage costs. Responses do not assert directory-provider certification. + +Errors use `urn:ietf:params:scim:api:messages:2.0:Error` with string `status`, `scimType` +and a bounded detail. With SCIM enabled, missing tables, account columns, security +revision storage, atomic collaborators or durable storage return JSON 503 before +account changes. Storage outages never fall back to local account writes or retry +uncertain mutations. The Worker forwards the public path, body, bearer and `If-Match` +to the Container; the private SCIM domain only accepts its fixed internal POST target. diff --git a/intelligence-migrations/0036_scim_provisioning.sql b/intelligence-migrations/0036_scim_provisioning.sql new file mode 100644 index 00000000..6001f7a1 --- /dev/null +++ b/intelligence-migrations/0036_scim_provisioning.sql @@ -0,0 +1,37 @@ +-- Tenant-scoped external identities, retained account mappings and SCIM revisions. +CREATE TABLE IF NOT EXISTS scim_resources ( + org_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('Users', 'Groups')), + id TEXT NOT NULL, + external_id TEXT, + user_name TEXT, + display_name TEXT, + deactivated INTEGER NOT NULL DEFAULT 0 CHECK(deactivated IN (0, 1)), + revision INTEGER NOT NULL CONSTRAINT scim_cas_confirmed CHECK(revision BETWEEN 1 AND 9007199254740991), + document TEXT NOT NULL, + PRIMARY KEY(org_id, kind, id), + UNIQUE(org_id, kind, external_id), + UNIQUE(org_id, kind, user_name) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_scim_account_identity ON scim_resources(user_name) WHERE kind='Users'; +CREATE TABLE IF NOT EXISTS scim_group_members ( + org_id TEXT NOT NULL, + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + PRIMARY KEY(org_id, group_id, user_id) +); +CREATE TABLE IF NOT EXISTS scim_token_digests ( + org_id TEXT PRIMARY KEY, + digest_sha256 TEXT NOT NULL CHECK(length(digest_sha256)=64), + updated_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS scim_audit ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + org_id TEXT NOT NULL, + kind TEXT NOT NULL, + resource_id TEXT NOT NULL, + revision INTEGER NOT NULL, + operation TEXT NOT NULL, + at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_scim_audit_scope ON scim_audit(org_id, sequence); diff --git a/package.json b/package.json index c66e4a8f..8d3b0463 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_scim_provisioning_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/scim.py b/routes/scim.py new file mode 100644 index 00000000..9fada8c5 --- /dev/null +++ b/routes/scim.py @@ -0,0 +1,145 @@ +"""SCIM bearer API with no dashboard or cookie-based authorisation.""" +import json + +from flask import Blueprint, Response, current_app, g, request + +from services import scim_provisioning as scim + + +def response(body=None, status=200, resource=None): + result = Response("" if body is None else json.dumps(body, ensure_ascii=True), status=status, + content_type="application/scim+json") + result.headers["Cache-Control"] = "no-store" + if resource: + result.headers["ETag"] = resource["meta"]["version"] + if status == 201: + result.headers["Location"] = resource["meta"]["location"] + return result + + +def discovery(name, resource_id=None): + definitions = [] + for kind, schema, attributes in (("User", scim.USER_SCHEMA, scim.USER_ATTRIBUTES), + ("Group", scim.GROUP_SCHEMA, scim.GROUP_ATTRIBUTES)): + if name == "Schemas": + definitions.append({"schemas": ["urn:ietf:params:scim:schemas:core:2.0:Schema"], + "id": schema, "name": kind, "attributes": _attributes(kind, attributes)}) + else: + definitions.append({"schemas": ["urn:ietf:params:scim:schemas:core:2.0:ResourceType"], + "id": kind, "name": kind, "endpoint": "/" + kind + "s", "schema": schema, + "schemaExtensions": []}) + if name == "ServiceProviderConfig": + if resource_id: + raise scim.ScimError(404, "Resource not found") + return {"schemas": ["urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig"], + "patch": {"supported": True}, "bulk": {"supported": False, "maxOperations": 0, "maxPayloadSize": 0}, + "filter": {"supported": True, "maxResults": 100}, "changePassword": {"supported": False}, + "sort": {"supported": False}, "etag": {"supported": True}, + "authenticationSchemes": [{"type": "oauthbearertoken", "name": "Bearer token", + "description": "Tenant-bound provisioning bearer token", "primary": True}]} + if resource_id: + for definition in definitions: + if definition["id"] == resource_id: + return definition + raise scim.ScimError(404, "Resource not found") + return {"schemas": [scim.LIST_SCHEMA], "totalResults": len(definitions), + "startIndex": 1, "itemsPerPage": len(definitions), "Resources": definitions} + + +def _attributes(kind, attributes): + values = [] + for name in sorted(attributes): + complex_attribute = name in {"name", "emails", "members"} + value = {"name": name, "type": "complex" if complex_attribute else "boolean" if name == "active" else "string", + "multiValued": name in {"emails", "members"}, "required": name == ("userName" if kind == "User" else "displayName"), + "mutability": "immutable" if name == "userName" else "readWrite", "returned": "default", + "uniqueness": "server" if name in {"externalId", "userName"} else "none"} + if complex_attribute: + names = {"name": ["formatted", "givenName", "familyName", "middleName", "honorificPrefix", "honorificSuffix"], + "emails": ["value", "type", "display", "primary"], "members": ["value", "display", "$ref"]}[name] + value["subAttributes"] = [{"name": key, "type": "boolean" if key == "primary" else "string", + "multiValued": False, "required": key == "value", "mutability": "readWrite", + "returned": "default", "uniqueness": "none"} for key in names] + values.append(value) + return values + + +def register_scim_routes(app, *, service=None): + """Mount explicit collaborators; an enabled unconfigured authority fails closed.""" + if service is not None: + app.extensions["scim_service"] = service + blueprint = Blueprint("scim", __name__, url_prefix="/scim/v2") + + def guard(): + if not request.path.startswith("/scim/v2/"): + return None + if not scim.enabled(): + return response(scim.ScimError(404, "Resource not found").body(), 404) + try: + g.scim_scope = scim.authenticate(request.headers.get("Authorization")) + if request.method not in {"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"}: + raise scim.ScimError(405, "Method not allowed") + if not request.endpoint or not request.endpoint.startswith("scim."): + raise scim.ScimError(404, "Resource not found") + except scim.ScimError as error: + return response(error.body(), error.status) + return None + + app.before_request_funcs.setdefault(None, []).insert(0, guard) + + @blueprint.route("/", methods=["GET", "POST", "PUT", "PATCH", "DELETE"], provide_automatic_options=False) + @blueprint.route("//", methods=["GET", "POST", "PUT", "PATCH", "DELETE"], provide_automatic_options=False) + def scim_resource(kind, resource_id=None): + try: + org, digest = g.scim_scope + authority = current_app.extensions.get("scim_service") + if authority is None: + raise scim.ScimError(503, "SCIM authority unavailable") + authority.ready() + if kind in {"Schemas", "ResourceTypes", "ServiceProviderConfig"}: + if request.method not in {"GET", "HEAD"}: + raise scim.ScimError(405, "Method not allowed") + return response(discovery(kind, resource_id)) + if kind not in {"Users", "Groups"}: + raise scim.ScimError(404, "Resource not found") + return _resource_request(authority, org, digest, kind, resource_id) + except scim.ScimError as error: + return response(error.body(), error.status) + except Exception: + return response(scim.ScimError(503, "SCIM authority unavailable").body(), 503) + + csrf = app.extensions.get("csrf") + if csrf is not None: + csrf.exempt(blueprint) + app.register_blueprint(blueprint) + + +def _resource_request(authority, org, digest, kind, resource_id): + if request.method in {"GET", "HEAD"}: + if resource_id is None: + return response(authority.list(org, kind, request.args)) + resource = authority.get(org, kind, resource_id) + return response(resource, resource=resource) + body = None + if request.method != "DELETE": + if request.mimetype not in {"application/json", "application/scim+json"}: + raise scim.ScimError(400, "JSON content type required", "invalidSyntax") + if request.content_length is not None and request.content_length > scim.MAX_BODY: + raise scim.ScimError(413, "SCIM body limit exceeded", "tooMany") + raw = request.stream.read(scim.MAX_BODY + 1) + if len(raw) > scim.MAX_BODY: + raise scim.ScimError(413, "SCIM body limit exceeded", "tooMany") + try: + body = json.loads(raw) + except (ValueError, UnicodeError): + raise scim.ScimError(400, "Invalid JSON", "invalidSyntax") from None + if request.method == "POST" and resource_id is None: + resource, created = authority.create(org, kind, body, digest) + return response(resource, 201 if created else 200, resource) + if request.method in {"PUT", "PATCH", "DELETE"} and resource_id is not None: + resource = authority.update(org, kind, resource_id, body, patch=request.method == "PATCH", + delete=request.method == "DELETE", if_match=request.headers.get("If-Match"), + token_digest=digest) + return response(None if request.method == "DELETE" else resource, + 204 if request.method == "DELETE" else 200, resource) + raise scim.ScimError(405, "Method not allowed") diff --git a/services/scim_provisioning.py b/services/scim_provisioning.py new file mode 100644 index 00000000..416f22c2 --- /dev/null +++ b/services/scim_provisioning.py @@ -0,0 +1,444 @@ +"""Strict SCIM resources and tenant-bound atomic persistence collaborators.""" +from __future__ import annotations + +import copy +import hashlib +import hmac +import json +import logging +import os +import re +import uuid +from datetime import datetime, timezone +from functools import lru_cache + +from error_handlers import APIError +from services.auth_primitives import build_api_key_prefix, hash_api_key, serialize_datetime, serialize_scopes +from services.enterprise_contract import AuthorityDenied, AuthorityOperation, TenantContext +from services.user_provisioning import provision_scim_user +from services.user_store import USER_FIELDS + +logger = logging.getLogger(__name__) +USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" +GROUP_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:Group" +PATCH_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:PatchOp" +ERROR_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:Error" +LIST_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:ListResponse" +MAX_BODY = 131072 +USER_ATTRIBUTES = {"userName", "active", "displayName", "name", "emails", "externalId"} +GROUP_ATTRIBUTES = {"displayName", "externalId", "members"} +FILTER = re.compile(r'(userName|externalId|displayName) eq ("(?:[^"\\]|\\.)*")\Z') +MEMBER_PATH = re.compile(r'members\[value eq ("(?:[^"\\]|\\.)*")\]\Z') +REF = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}\Z") + + +class ScimError(Exception): + def __init__(self, status, detail, scim_type="invalidValue"): + super().__init__(detail) + self.status, self.detail, self.scim_type = status, detail, scim_type + + def body(self): + return {"schemas": [ERROR_SCHEMA], "status": str(self.status), + "scimType": self.scim_type, "detail": self.detail} + + +@lru_cache(maxsize=1) +def _warn_once(): + logger.warning("Invalid SCIM_ENABLED; SCIM provisioning disabled") + + +def enabled(env=None): + raw = (os.environ if env is None else env).get("SCIM_ENABLED", "") + flag = raw.strip().lower() if isinstance(raw, str) else "invalid" + if flag in {"", "0", "false", "off", "no"}: + return False + if flag in {"1", "true", "on", "yes"}: + return True + _warn_once() + return False + + +def authenticate(authorization, env=None): + """Only an unambiguous constant-time digest match establishes tenant scope.""" + env = os.environ if env is None else env + if not isinstance(authorization, str) or not re.fullmatch(r"Bearer [^\s]{1,4096}", authorization, re.I): + raise ScimError(401, "Provisioning bearer token required") + presented = hashlib.sha256(authorization.split(" ", 1)[1].encode()).digest() + try: + config = json.loads(env.get("SCIM_TRUST_CONFIG_JSON", "") or "{}") + if not isinstance(config, dict) or set(config) != {"tenants"} or not isinstance(config["tenants"], dict): + raise ValueError() + if len(config["tenants"]) > 1000: + raise ValueError() + matches = [] + for org, entry in config["tenants"].items(): + TenantContext("scim", org) + if not isinstance(entry, dict) or set(entry) != {"token_ref"} or not isinstance(entry["token_ref"], str): + raise ValueError() + if not REF.fullmatch(entry["token_ref"]): + raise ValueError() + token = env.get(entry["token_ref"]) + if not isinstance(token, str) or not token or len(token) > 4096: + continue + if hmac.compare_digest(presented, hashlib.sha256(token.encode()).digest()): + matches.append(org) + if len(matches) != 1: + raise ValueError() + return matches[0], presented.hex() + except (ValueError, TypeError, AttributeError): + raise ScimError(401, "Provisioning bearer token not recognised") from None + + +def text(value, maximum=1024): + if (not isinstance(value, str) or not value or len(value) > maximum + or any(ord(c) < 32 or ord(c) == 127 or 0xD800 <= ord(c) <= 0xDFFF for c in value)): + raise ScimError(400, "Invalid string attribute") + return value + + +def validate_resource(body, kind): + if not isinstance(body, dict): + raise ScimError(400, "Resource must be an object", "invalidSyntax") + attributes = USER_ATTRIBUTES if kind == "Users" else GROUP_ATTRIBUTES + if set(body) - attributes - {"schemas", "id", "meta"}: + raise ScimError(400, "Unsupported resource attribute") + schema = USER_SCHEMA if kind == "Users" else GROUP_SCHEMA + if "schemas" in body and body["schemas"] != [schema]: + raise ScimError(400, "Unsupported resource schema") + value = copy.deepcopy({key: item for key, item in body.items() if key in attributes}) + if "id" in body and not re.fullmatch(r"[a-f0-9]{32}", text(body["id"], 128)): + raise ScimError(400, "Invalid resource id") + if "meta" in body: + meta = body["meta"] + if not isinstance(meta, dict) or set(meta) - {"resourceType", "created", "lastModified", "location", "version"}: + raise ScimError(400, "Unsupported metadata attribute") + for item in meta.values(): + text(item) + for key in ("externalId", "displayName"): + if key in value: + text(value[key]) + if kind == "Groups": + text(value.get("displayName")) + validate_members(value.setdefault("members", [])) + return value + username = text(value.get("userName"), 128) + if username.strip() != username: + raise ScimError(400, "userName must not have surrounding whitespace") + value.setdefault("active", True) + if type(value["active"]) is not bool: + raise ScimError(400, "active must be a boolean") + if "name" in value: + name = value["name"] + if not isinstance(name, dict) or set(name) - {"formatted", "familyName", "givenName", "middleName", "honorificPrefix", "honorificSuffix"}: + raise ScimError(400, "Unsupported name attribute") + for item in name.values(): + text(item) + if "emails" in value: + emails = value["emails"] + if not isinstance(emails, list) or len(emails) > 100: + raise ScimError(400, "Invalid emails") + primary = 0 + for email in emails: + if not isinstance(email, dict) or set(email) - {"value", "type", "primary", "display"}: + raise ScimError(400, "Unsupported email attribute") + text(email.get("value")) + for key in ("type", "display"): + if key in email: + text(email[key]) + if "primary" in email and type(email["primary"]) is not bool: + raise ScimError(400, "Invalid primary email flag") + primary += email.get("primary", False) + if primary > 1: + raise ScimError(400, "Only one primary email is supported") + return value + + +def validate_members(members): + if not isinstance(members, list) or len(members) > 1000: + raise ScimError(400, "Group membership limit is 1000") + seen = set() + for member in members: + if not isinstance(member, dict) or set(member) - {"value", "display", "$ref"}: + raise ScimError(400, "Unsupported member attribute") + member_id = text(member.get("value"), 128) + if member_id in seen: + raise ScimError(400, "Duplicate group member") + seen.add(member_id) + for key in ("display", "$ref"): + if key in member: + text(member[key]) + + +def pagination(args): + try: + start, count = args.get("startIndex", "1"), args.get("count", "100") + if not re.fullmatch(r"[0-9]{1,9}", start) or not re.fullmatch(r"[0-9]{1,9}", count) or int(start) < 1: + raise ValueError() + return int(start), min(int(count), 100) + except (TypeError, ValueError): + raise ScimError(400, "Invalid pagination") from None + + +def parse_filter(expression): + if expression is None: + return None, None + match = FILTER.fullmatch(expression) if len(expression) <= 2048 else None + if not match: + raise ScimError(400, "Only equality on userName, externalId or displayName is supported", "invalidFilter") + try: + return match[1], text(json.loads(match[2])) + except (ValueError, TypeError): + raise ScimError(400, "Invalid filter string", "invalidFilter") from None + + +def revision(resource): + return int(resource["meta"]["version"][3:-1]) + + +def match_version(resource, if_match): + if if_match is not None and if_match not in {"*", resource["meta"]["version"]}: + raise ScimError(412, "Resource version does not match") + + +def patch_resource(resource, body, kind): + if (not isinstance(body, dict) or set(body) != {"schemas", "Operations"} + or body["schemas"] != [PATCH_SCHEMA] or not isinstance(body["Operations"], list) + or not 1 <= len(body["Operations"]) <= 100): + raise ScimError(400, "Invalid PATCH document", "invalidSyntax") + result = copy.deepcopy(resource) + for operation in body["Operations"]: + _patch_operation(result, operation, kind) + return validate_resource(result, kind) + + +def _patch_operation(result, operation, kind): + if not isinstance(operation, dict) or set(operation) - {"op", "path", "value"}: + raise ScimError(400, "Invalid PATCH operation", "invalidSyntax") + op, path = operation.get("op"), operation.get("path") + if not isinstance(op, str) or op.lower() not in {"add", "replace", "remove"}: + raise ScimError(400, "Unsupported PATCH operation", "invalidSyntax") + op = op.lower() + if path is None and op in {"add", "replace"} and isinstance(operation.get("value"), dict): + for key, value in operation["value"].items(): + _patch_operation(result, {"op": op, "path": key, "value": value}, kind) + return + allowed = (USER_ATTRIBUTES - {"userName"}) if kind == "Users" else GROUP_ATTRIBUTES + selector = MEMBER_PATH.fullmatch(path) if isinstance(path, str) and kind == "Groups" else None + if selector and op == "remove": + if "value" in operation: + raise ScimError(400, "remove does not accept value", "invalidSyntax") + try: + member_id = text(json.loads(selector[1]), 128) + except (TypeError, ValueError): + raise ScimError(400, "Invalid member selector", "invalidSyntax") from None + result["members"] = [member for member in result.get("members", []) if member["value"] != member_id] + return + if not isinstance(path, str) or path not in allowed: + raise ScimError(400, "Unsupported PATCH path", "invalidSyntax") + if op == "remove": + if "value" in operation: + raise ScimError(400, "remove does not accept value", "invalidSyntax") + result.pop(path, None) + if path == "active": + result["active"] = False + return + if "value" not in operation: + raise ScimError(400, "PATCH value required", "invalidSyntax") + value = copy.deepcopy(operation["value"]) + if op == "add" and path in {"members", "emails"}: + if not isinstance(value, list): + raise ScimError(400, "Multi-valued attribute requires an array") + value = result.get(path, []) + value + if path == "members": + unique = {} + for member in value: + validate_members([member]) + unique[member["value"]] = member + value = list(unique.values()) + elif op == "add" and path == "name" and isinstance(value, dict): + value = {**result.get(path, {}), **value} + result[path] = value + + +def _account_row(**values): + row = {key: None for key in USER_FIELDS} + for key, value in values.items(): + if key == "controls": + row.update(value or {}) + elif key == "scopes": + row[key] = serialize_scopes(value) + elif key == "is_admin": + row[key] = int(value) + elif key in USER_FIELDS: + row[key] = serialize_datetime(value) if isinstance(value, datetime) else value + return row + + +class D1ScimStore: + """One-shot private RPC; storage failure never falls back to local accounts.""" + def __init__(self, call): + self.call = call + + def _call(self, operation, **values): + try: + body = self.call({"version": 1, "operation": operation, **values}) + except Exception as error: + status = getattr(error, "status", 503) + if status in {400, 409, 412}: + raise ScimError(status, "SCIM storage rejected the operation", + "uniqueness" if status == 409 else "invalidValue") from None + raise ScimError(503, "SCIM storage is unavailable") from None + if not isinstance(body, dict) or body.get("version") != 1 or "error" in body: + raise ScimError(503, "Invalid SCIM storage response") + return body + + def probe(self): + if (os.environ.get("AUTH_STORAGE_BACKEND", "").strip().lower() != "d1" + or os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip().lower() != "d1" + or os.environ.get("CONFIG_REVISION_SYNC_ENABLED", "").strip().lower() not in {"true", "1", "yes", "on"}): + raise ScimError(503, "SCIM requires durable accounts and security revision sync") + self._call("probe") + + def get(self, org, kind, resource_id): + return self._call("get", org_id=org, kind=kind, id=resource_id)["resource"] + + def account(self, username): + return self._call("account", username=username)["account"] + + def list(self, org, kind, attribute=None, value=None, start=1, count=100): + body = self._call("list", org_id=org, kind=kind, attribute=attribute, value=value, start=start, count=count) + return body["resources"], body["total"] + + def put(self, org, kind, resource, expected, account=None, token_digest=None, deactivate=False): + body = self._call("put", org_id=org, kind=kind, resource=resource, expected=expected, + account=account, token_digest=token_digest, deactivate=deactivate) + return body["resource"], body["created"] + + +class ScimService: + def __init__(self, store, auth, *, tenant_authority=None): + self.store, self.auth, self.tenant_authority = store, auth, tenant_authority + + def ready(self): + self.store.probe() + + def _group_scope(self, org, resource): + if self.tenant_authority is None: + raise ScimError(503, "Tenant authority unavailable") + context = TenantContext("scim", org, resource["id"]) + operation = AuthorityOperation(context, resource["id"], revision(resource), uuid.uuid4().hex) + try: + granted = self.tenant_authority(operation) + except AuthorityDenied: + raise ScimError(503, "Tenant authority unavailable") from None + if type(granted) is not TenantContext or granted != context: + raise ScimError(403, "Tenant authority scope mismatch") + for member in resource.get("members", []): + user = self.store.get(org, "Users", member["value"]) + if user is None: + raise ScimError(400, "Member must belong to this organisation") + member_context = TenantContext(user["userName"], org) + try: + decision = self.tenant_authority(AuthorityOperation(member_context, member["value"], 0, uuid.uuid4().hex)) + except AuthorityDenied: + raise ScimError(400, "Member must belong to this organisation") from None + if type(decision) is not TenantContext or decision != member_context: + raise ScimError(400, "Member must belong to this organisation") + + def get(self, org, kind, resource_id): + self.ready() + resource = self.store.get(org, kind, resource_id) + if resource is None: + raise ScimError(404, "Resource not found") + return resource + + def list(self, org, kind, args): + self.ready() + if set(args) - {"filter", "startIndex", "count"}: + raise ScimError(400, "Unsupported list parameter", "invalidSyntax") + start, count = pagination(args) + attribute, value = parse_filter(args.get("filter")) + rows, total = self.store.list(org, kind, attribute, value, start, count) + return {"schemas": [LIST_SCHEMA], "totalResults": total, "startIndex": start, + "itemsPerPage": len(rows), "Resources": rows} + + def create(self, org, kind, body, token_digest=None): + self.ready() + value = validate_resource(body, kind) + if value.get("externalId"): + rows, _ = self.store.list(org, kind, "externalId", value["externalId"], 1, 1) + if rows: + if kind == "Groups": + self._group_scope(org, rows[0]) + return rows[0], False + resource = self._resource(value, kind) + account = None + if kind == "Users": + prepared = [] + try: + provision_scim_user(self.auth, value["userName"], active=value["active"], + persist=lambda **data: prepared.append(_account_row(**data))) + except APIError as error: + if error.status_code == 409 and value.get("externalId"): + rows, _ = self.store.list(org, kind, "externalId", value["externalId"], 1, 1) + if rows: + return rows[0], False + raise ScimError(error.status_code, "Account provisioning rejected", + "uniqueness" if error.status_code == 409 else "invalidValue") from None + account = prepared[0] + else: + self._group_scope(org, resource) + result = self.store.put(org, kind, resource, 0, account, token_digest) + self.auth._forget_verified_keys() + return result + + @staticmethod + def _resource(value, kind, old=None): + now = datetime.now(timezone.utc).isoformat() + resource_id = old["id"] if old else uuid.uuid4().hex + return {"schemas": [USER_SCHEMA if kind == "Users" else GROUP_SCHEMA], "id": resource_id, **value, + "meta": {"resourceType": "User" if kind == "Users" else "Group", + "version": f'W/"{revision(old) + 1 if old else 1}"', + "created": old["meta"]["created"] if old else now, "lastModified": now, + "location": f"/scim/v2/{kind}/{resource_id}"}} + + def update(self, org, kind, resource_id, body, *, patch=False, delete=False, if_match=None, token_digest=None): + old = self.get(org, kind, resource_id) + match_version(old, if_match) + if delete: + value = validate_resource(old, kind) + if kind == "Users": + value["active"] = False + else: + value["members"] = [] + else: + if isinstance(body, dict) and "id" in body and body["id"] != resource_id: + raise ScimError(400, "id is immutable", "mutability") + value = patch_resource(old, body, kind) if patch else validate_resource(body, kind) + if kind == "Users" and value["userName"] != old["userName"]: + raise ScimError(400, "userName is immutable", "mutability") + resource = self._resource(value, kind, old) + account = self._activation_account(old, resource) if kind == "Users" else None + if kind == "Groups": + self._group_scope(org, resource) + saved, _ = self.store.put(org, kind, resource, revision(old), account, token_digest, + deactivate=delete and kind == "Groups") + self.auth._forget_verified_keys() + return saved + + def _activation_account(self, old, resource): + if old["active"] == resource["active"]: + return None + account = self.store.account(old["userName"]) + if not account or account["is_admin"] != 0: + raise ScimError(503, "Provisioned account unavailable") + if resource["active"] is False: + key = self.auth._generate_api_key() + # Sessions compare the short prefix; a collision must not resurrect them. + if build_api_key_prefix(key) == account["api_key_prefix"]: + key = ("A" if key[0] != "A" else "B") + key[1:] + account.update(api_key_hash=hash_api_key(key), api_key_prefix=build_api_key_prefix(key), + revoked_at=datetime.now(timezone.utc).isoformat()) + else: + account["revoked_at"] = None + return account diff --git a/services/user_provisioning.py b/services/user_provisioning.py index 7c6e71ec..98f8887b 100644 --- a/services/user_provisioning.py +++ b/services/user_provisioning.py @@ -1,10 +1,11 @@ """Account provisioning policy shared by the existing Access console.""" +import os from datetime import datetime, timezone from error_handlers import APIError from services.auth_primitives import ( - build_api_key_prefix, default_scopes, require_valid_username, serialize_datetime, + build_api_key_prefix, default_scopes, hash_api_key, require_valid_username, serialize_datetime, ) from services.intelligence_auth import reject_local_integration_management @@ -43,3 +44,33 @@ def create_user(auth, username, is_admin=False, scopes=None): "api_key_prefix": build_api_key_prefix(api_key), "scopes": list(granted_scopes), "is_admin": is_admin, "created_at": serialize_datetime(created_at), "last_login": None, } + + +def provision_scim_user(auth, username, *, is_admin=False, scopes=None, active=True, persist=None): + """Prepare a least-privilege account for an atomic provisioning authority. + + The generated credential is discarded. A trusted persistence callback can commit + the account and its external identity together without an interactive session. + """ + if is_admin is not False or (scopes is not None and scopes != list(default_scopes(False))): + raise APIError("SCIM cannot grant administrator or elevated scopes", 400) + if type(active) is not bool: + raise APIError("SCIM active must be a boolean", 400) + username = require_valid_username(username) + reject_local_integration_management(username) + reserved = {os.environ.get("ADMIN_USERNAME", "admin").strip(), + *(name.strip() for name in os.environ.get("ADMIN_USERNAMES", "").split(","))} + if username in reserved: + raise APIError("SCIM cannot provision an environment-managed administrator", 400) + if auth._load_user_by_username(username) is not None: + raise APIError("User already exists", 409) + api_key = auth._generate_api_key() + now = datetime.now(timezone.utc) + (persist or auth._persist_user)( + username=username, api_key_hash=hash_api_key(api_key), + api_key_prefix=build_api_key_prefix(api_key), is_admin=False, + created_at=now, last_login=None, scopes=default_scopes(False), + created_by="scim", revoked_at=None if active else now, + ) + return {"id": username, "username": username, "is_admin": False, + "scopes": list(default_scopes(False))} diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..01dd2638 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -161,6 +161,10 @@ "routes/responses_state.py": [ "app.route('/v1/responses/', methods=['GET', 'DELETE'])" ], + "routes/scim.py": [ + "blueprint.route('/', methods=['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], provide_automatic_options=False)", + "blueprint.route('//', methods=['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], provide_automatic_options=False)" + ], "routes/shadow_eval.py": [ "app.get('/admin/workbench/shadow/league')", "app.get('/admin/workbench/shadow/samples')", @@ -399,6 +403,7 @@ "worker/roleplay/validation.mjs", "worker/route-health-d1.mjs", "worker/scheduled-maintenance.mjs", + "worker/scim-d1.mjs", "worker/secret-digest.mjs", "worker/secret-firewall.mjs", "worker/secret-scan.mjs", diff --git a/tests/test_scim_provisioning.py b/tests/test_scim_provisioning.py new file mode 100644 index 00000000..848b1590 --- /dev/null +++ b/tests/test_scim_provisioning.py @@ -0,0 +1,376 @@ +"""SCIM lifecycle through registered bearer routes with isolated authorities.""" +import copy +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import Mock + +import pytest +from flask import Flask, Response +from flask_wtf.csrf import CSRFProtect + +from services import scim_provisioning as scim +from services import user_provisioning as provisioning +from routes.scim import register_scim_routes + + +class Store: + def __init__(self): + self.rows = {} + self.accounts = {} + self.audit = [] + self.lock = threading.RLock() + self.available = True + + def probe(self): + if not self.available: + raise scim.ScimError(503, "Storage unavailable") + + def list(self, org, kind, attribute=None, value=None, start=1, count=100): + rows = [copy.deepcopy(row) for (tenant, resource_kind, _), row in self.rows.items() + if tenant == org and resource_kind == kind] + rows.sort(key=lambda row: row["id"]) + if attribute: + rows = [row for row in rows if row.get(attribute) == value] + return rows[start - 1:start - 1 + count], len(rows) + + def get(self, org, kind, resource_id): + return copy.deepcopy(self.rows.get((org, kind, resource_id))) + + def account(self, username): + return copy.deepcopy(self.accounts.get(username)) + + def put(self, org, kind, resource, expected, account=None, token_digest=None, deactivate=False): + with self.lock: + self.probe() + key = (org, kind, resource["id"]) + old = self.rows.get(key) + if expected == 0: + for (tenant, resource_kind, _), row in self.rows.items(): + if tenant == org and resource_kind == kind: + if resource.get("externalId") and row.get("externalId") == resource["externalId"]: + return copy.deepcopy(row), False + if kind == "Users" and row["userName"] == resource["userName"]: + raise scim.ScimError(409, "Duplicate userName", "uniqueness") + if account and account["username"] in self.accounts: + raise scim.ScimError(409, "Duplicate account", "uniqueness") + elif not old or scim.revision(old) != expected: + raise scim.ScimError(412, "Stale version", "invalidVers") + self.rows[key] = copy.deepcopy(resource) + if account: + self.accounts[account["username"]] = copy.deepcopy(account) + self.audit.append((org, kind, resource["id"], expected + 1)) + return copy.deepcopy(resource), expected == 0 + + +@pytest.fixture +def setup(monkeypatch): + monkeypatch.setenv("ADMIN_USERNAME", "admin") + monkeypatch.setenv("ADMIN_USERNAMES", "") + monkeypatch.setenv("SCIM_ENABLED", "true") + monkeypatch.setenv("SCIM_TRUST_CONFIG_JSON", json.dumps({"tenants": { + "org:a": {"token_ref": "SCIM_TEST_A"}, "org:b": {"token_ref": "SCIM_TEST_B"}}})) + monkeypatch.setenv("SCIM_TEST_A", "synthetic-scim-a") + monkeypatch.setenv("SCIM_TEST_B", "synthetic-scim-b") + auth = Mock() + auth._load_user_by_username.return_value = None + auth._generate_api_key.return_value = "synthetic-discarded-key" + store = Store() + authority = Mock(side_effect=lambda operation: operation.context) + service = scim.ScimService(store, auth, tenant_authority=authority) + app = Flask(__name__) + app.secret_key = "synthetic-csrf" + CSRFProtect(app) + app.add_url_rule("/raw", view_func=lambda: Response(b"unchanged", headers={"X-Raw": "same"})) + register_scim_routes(app, service=service) + return app.test_client(), store, auth, authority + + +def call(client, method, path="Users", body=None, tenant="a", **headers): + return client.open("/scim/v2/" + path, method=method, json=body, + headers={"Authorization": "Bearer synthetic-scim-" + tenant, **headers}) + + +def user(client, username="alice", external="subject-a", **changes): + return call(client, "POST", body={"schemas": [scim.USER_SCHEMA], "userName": username, + "externalId": external, **changes}) + + +@pytest.mark.parametrize("flag", ["", "false", "0", "off", "bad-private-value"]) +def test_disabled_all_paths_precede_auth_and_storage(setup, monkeypatch, flag): + client, store, auth, _ = setup + monkeypatch.setenv("SCIM_ENABLED", flag) + store.available = False + for method in ("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"): + assert client.open("/scim/v2/unknown", method=method).status_code == 404 + assert client.get("/raw").data == b"unchanged" + auth._load_user_by_username.assert_not_called() + + +def test_flag_warns_once_without_value(caplog): + scim._warn_once.cache_clear() + assert not scim.enabled({}) + for _ in range(2): + assert not scim.enabled({"SCIM_ENABLED": "private-invalid"}) + assert caplog.text.count("Invalid SCIM_ENABLED") == 1 + assert "private-invalid" not in caplog.text + + +def test_auth_scope_discovery_and_csrf_exemption(setup): + client, _, _, _ = setup + for headers in ({}, {"Authorization": "Bearer unknown"}, {"Authorization": "Basic x"}): + response = client.get("/scim/v2/Users", headers=headers) + assert response.status_code == 401 + assert response.json["schemas"] == [scim.ERROR_SCHEMA] + assert response.json["status"] == "401" + assert response.mimetype == "application/scim+json" + for endpoint in ("ServiceProviderConfig", "Schemas", "ResourceTypes"): + assert call(client, "GET", endpoint).status_code == 200 + assert call(client, "GET", "Schemas/" + scim.USER_SCHEMA).status_code == 200 + response = user(client) + assert response.status_code == 201 + assert response.headers["ETag"] == response.json["meta"]["version"] + assert response.headers["Location"].endswith("/Users/" + response.json["id"]) + assert "synthetic-discarded-key" not in response.get_data(as_text=True) + assert call(client, "GET", "Users/" + response.json["id"], tenant="b").status_code == 404 + + +def test_idempotency_uniqueness_default_scopes_and_no_admin(setup): + client, store, auth, _ = setup + first = user(client) + repeat = user(client, username="renamed") + assert repeat.status_code == 200 and repeat.json == first.json + assert len(store.accounts) == len(store.audit) == 1 + account = store.accounts["alice"] + assert account["is_admin"] == 0 and account["scopes"] == "chat,models" + auth._require_admin.assert_not_called() + assert user(client, external="other").status_code == 409 + assert user(client, username="admin", external="admin").status_code == 400 + for changes in ({"is_admin": True}, {"roles": [{"value": "admin"}]}, {"scopes": ["admin"]}): + assert user(client, username="evil", external="evil", **changes).status_code == 400 + for kwargs in ({"is_admin": True}, {"scopes": ["admin"]}, {"scopes": ["knowledge:manage"]}): + with pytest.raises(Exception): + provisioning.provision_scim_user(auth, "evil", **kwargs) + + +def test_patch_etag_deactivate_reactivate_preserves_history(setup): + client, store, auth, _ = setup + created = user(client) + path = "Users/" + created.json["id"] + old_account = copy.deepcopy(store.accounts["alice"]) + patch = {"schemas": [scim.PATCH_SCHEMA], "Operations": [ + {"op": "replace", "path": "displayName", "value": "Alice"}, + {"op": "add", "path": "emails", "value": [{"value": "a@example.test", "primary": True}]}, + {"op": "replace", "path": "active", "value": False}]} + auth._generate_api_key.return_value = "changed-synthetic-discarded-key" + changed = call(client, "PATCH", path, patch, **{"If-Match": created.headers["ETag"]}) + assert changed.status_code == 200 and changed.json["active"] is False + assert store.accounts["alice"]["revoked_at"] + assert store.accounts["alice"]["api_key_prefix"] != old_account["api_key_prefix"] + assert store.accounts["alice"]["created_at"] == old_account["created_at"] + for method in ("PUT", "PATCH", "DELETE"): + assert call(client, method, path, patch, **{"If-Match": created.headers["ETag"]}).status_code == 412 + prefix = store.accounts["alice"]["api_key_prefix"] + on = {"schemas": [scim.PATCH_SCHEMA], "Operations": [{"op": "replace", "path": "active", "value": True}]} + assert call(client, "PATCH", path, on).status_code == 200 + assert store.accounts["alice"]["api_key_prefix"] == prefix + assert store.accounts["alice"]["revoked_at"] is None + assert call(client, "DELETE", path).status_code == 204 + assert call(client, "GET", path).json["externalId"] == "subject-a" + assert len(store.accounts) == 1 + + +def test_filter_pagination_and_strict_validation(setup): + client, _, _, _ = setup + user(client) + for expression in ('userName eq "alice"', 'externalId eq "subject-a"', 'displayName eq "missing"'): + response = call(client, "GET", "Users?filter=" + expression + "&count=101") + assert response.status_code == 200 and response.json["itemsPerPage"] <= 100 + for query in ("filter=active eq true", "filter=userName co alice", "startIndex=0", "count=-1", "count=bad"): + assert call(client, "GET", "Users?" + query).status_code == 400 + assert call(client, "GET", "Users?count=0").json["Resources"] == [] + for changes in ({"schemas": ["unknown"]}, {"active": "false"}, {"name": {"unknown": "x"}}, + {"emails": [{"value": "x", "unknown": True}]}): + assert user(client, **changes).status_code == 400 + path = "Users/" + user(client).json["id"] + for operation in ({"op": "move", "path": "active", "value": True}, + {"op": "replace", "path": "roles", "value": []}, + {"op": "replace", "path": "userName", "value": "bob"}): + assert call(client, "PATCH", path, {"schemas": [scim.PATCH_SCHEMA], "Operations": [operation]}).status_code == 400 + + +def test_groups_authority_members_and_patch_are_scoped(setup): + client, store, _, authority = setup + member = user(client).json["id"] + foreign = call(client, "POST", body={"userName": "bob", "externalId": "foreign"}, tenant="b").json["id"] + payload = {"schemas": [scim.GROUP_SCHEMA], "displayName": "engineering", "externalId": "team-a", + "members": [{"value": member}]} + created = call(client, "POST", "Groups", payload) + assert created.status_code == 201 + assert authority.call_args[0][0].context.org_id == "org:a" + path = "Groups/" + created.json["id"] + for value in (foreign, "unknown"): + bad = {"schemas": [scim.PATCH_SCHEMA], "Operations": [{"op": "add", "path": "members", "value": [{"value": value}]}]} + assert call(client, "PATCH", path, bad).status_code == 400 + removed = {"schemas": [scim.PATCH_SCHEMA], "Operations": [{"op": "remove", "path": 'members[value eq "' + member + '"]'}]} + assert call(client, "PATCH", path, removed).json["members"] == [] + assert call(client, "PUT", path, {"displayName": "renamed", "members": []}).status_code == 200 + assert call(client, "DELETE", path).status_code == 204 + assert len(store.rows) == 3 + authority.side_effect = lambda operation: scim.TenantContext("foreign", "org:b") + assert call(client, "POST", "Groups", payload).status_code == 403 + + +def test_missing_schema_and_authority_precede_account_changes(setup): + client, store, auth, authority = setup + store.available = False + response = user(client) + assert response.status_code == 503 and response.json["status"] == "503" + auth._generate_api_key.assert_not_called() + assert not store.accounts + store.available = True + authority.side_effect = scim.AuthorityDenied("unavailable") + assert call(client, "POST", "Groups", {"displayName": "team"}).status_code == 503 + assert not store.rows + + +def test_atomic_duplicate_create_and_stale_updates(setup): + _, store, auth, _ = setup + service = scim.ScimService(store, auth, tenant_authority=lambda op: op.context) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: service.create("org:a", "Users", {"userName": "alice", "externalId": "a"}), range(2))) + assert results[0][0]["id"] == results[1][0]["id"] + assert len(store.accounts) == 1 + + +def test_trust_config_ambiguity_and_invalid_values_fail_closed(monkeypatch): + env = {"SCIM_TRUST_CONFIG_JSON": json.dumps({"tenants": {"org:a": {"token_ref": "A"}, "org:b": {"token_ref": "B"}}}), + "A": "synthetic-shared", "B": "synthetic-shared"} + with pytest.raises(scim.ScimError) as error: + scim.authenticate("Bearer synthetic-shared", env) + assert error.value.status == 401 + for config in ("bad", "[]", '{"tenants":{"org:a":{"token":"raw"}}}'): + with pytest.raises(scim.ScimError): + scim.authenticate("Bearer synthetic-shared", {**env, "SCIM_TRUST_CONFIG_JSON": config}) + + +def test_real_auth_session_and_old_key_cannot_return_after_reactivation(setup, monkeypatch): + from flask import session + from services.auth_service import AuthService + from werkzeug.security import check_password_hash + + client, store, _, _ = setup + app = client.application + + class IsolatedAuth(AuthService): + _users = {} + _verified_keys = {} + _api_key_prefix_index = {} + + def lookup(username): + row = store.accounts.get(username) + return IsolatedAuth._row_to_user(row) if row else None + + monkeypatch.setattr(IsolatedAuth, "_load_user_by_username", staticmethod(lookup)) + monkeypatch.setattr(IsolatedAuth, "_generate_api_key", staticmethod(lambda: "original-synthetic-key")) + app.extensions["scim_service"].auth = IsolatedAuth + app.add_url_rule("/who", endpoint="session_identity", view_func=lambda: {"authenticated": IsolatedAuth.get_current_user() is not None}) + created = user(client) + assert created.status_code == 201 + original = copy.deepcopy(store.accounts["alice"]) + assert check_password_hash(original["api_key_hash"], "original-synthetic-key") + with client.session_transaction() as signed: + signed["authenticated"] = True + signed["user"] = {"username": "alice", "api_key_prefix": original["api_key_prefix"]} + assert client.get("/who").json["authenticated"] + monkeypatch.setattr(IsolatedAuth, "_generate_api_key", staticmethod(lambda: "original-synthetic-rotated-key")) + path = "Users/" + created.json["id"] + assert call(client, "DELETE", path).status_code == 204 + assert store.accounts["alice"]["api_key_prefix"] != original["api_key_prefix"] + assert client.get("/who").json["authenticated"] is False + patch = {"schemas": [scim.PATCH_SCHEMA], "Operations": [{"op": "replace", "path": "active", "value": True}]} + assert call(client, "PATCH", path, patch).status_code == 200 + assert not check_password_hash(store.accounts["alice"]["api_key_hash"], "original-synthetic-key") + with app.test_request_context("/who"): + session["authenticated"] = True + session["user"] = {"username": "alice", "api_key_prefix": original["api_key_prefix"]} + assert IsolatedAuth.get_current_user() is None + + +def test_transport_failure_has_no_retry_and_no_account_fallback(monkeypatch): + for name in ("AUTH_STORAGE_BACKEND", "INTELLIGENCE_STORAGE_BACKEND"): + monkeypatch.setenv(name, "d1") + monkeypatch.setenv("CONFIG_REVISION_SYNC_ENABLED", "true") + transport = Mock(side_effect=RuntimeError("private database details")) + store = scim.D1ScimStore(transport) + auth = Mock() + service = scim.ScimService(store, auth) + with pytest.raises(scim.ScimError) as error: + service.create("org:a", "Users", {"userName": "alice"}) + assert error.value.status == 503 + assert "private database details" not in json.dumps(error.value.body()) + transport.assert_called_once_with({"version": 1, "operation": "probe"}) + auth._persist_user.assert_not_called() + auth._generate_api_key.assert_not_called() + transport.reset_mock() + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "sql") + with pytest.raises(scim.ScimError) as error: + store.probe() + assert error.value.status == 503 + transport.assert_not_called() + + +def test_body_limit_unknown_metadata_and_partial_patch_do_not_mutate(setup): + client, store, _, _ = setup + created = user(client) + path = "Users/" + created.json["id"] + before = copy.deepcopy(store.rows) + for operations in ([{"op": "replace", "path": "displayName", "value": "changed"}, + {"op": "replace", "path": "roles", "value": ["admin"]}], + [{"op": "remove", "path": "members", "value": []}]): + assert call(client, "PATCH", path, {"schemas": [scim.PATCH_SCHEMA], "Operations": operations}).status_code == 400 + assert store.rows == before + assert user(client, meta={"unknown": "x"}).status_code == 400 + assert user(client, username=" alice ").status_code == 400 + raw = client.post("/scim/v2/Users", data="x" * (scim.MAX_BODY + 1), + headers={"Authorization": "Bearer synthetic-scim-a", "Content-Type": "application/scim+json"}) + assert raw.status_code == 413 and raw.json["status"] == "413" + invalid = client.post("/scim/v2/Users", data="invalid", headers={ + "Authorization": "Bearer synthetic-scim-a", "Content-Type": "application/scim+json"}) + assert invalid.status_code == 400 and invalid.json["scimType"] == "invalidSyntax" + for path in ("Users?sortBy=userName", "missing/nested/path"): + result = call(client, "GET", path) + assert result.status_code in {400, 404} and result.mimetype == "application/scim+json" + assert call(client, "HEAD", "Users").status_code == 200 + + +def test_successful_put_and_patch_all_mutable_attributes(setup): + client, _, _, _ = setup + created = user(client) + path = "Users/" + created.json["id"] + changed = call(client, "PUT", path, {"schemas": [scim.USER_SCHEMA], "userName": "alice", "externalId": "new-subject", + "displayName": "Alice", "name": {"givenName": "Alice"}, "emails": []}) + assert changed.status_code == 200 + patch = {"schemas": [scim.PATCH_SCHEMA], "Operations": [ + {"op": "add", "path": "name", "value": {"familyName": "Example"}}, + {"op": "replace", "path": "externalId", "value": "final-subject"}, + {"op": "remove", "path": "displayName"}, + {"op": "remove", "path": "emails"}]} + result = call(client, "PATCH", path, patch) + assert result.status_code == 200 and result.json["name"] == {"givenName": "Alice", "familyName": "Example"} + assert result.json["externalId"] == "final-subject" + assert "emails" not in result.json and "displayName" not in result.json + assert call(client, "PUT", path, {"userName": "bob"}).status_code == 400 + + +def test_registration_default_keeps_existing_bytes_and_headers(monkeypatch): + monkeypatch.setenv("SCIM_ENABLED", "") + def baseline(): + app = Flask(__name__) + app.add_url_rule("/raw", view_func=lambda: Response(b"opaque bytes", headers={"X-Raw": "identical"})) + return app + before = baseline().test_client().get("/raw") + app = baseline() + register_scim_routes(app) + after = app.test_client().get("/raw") + assert (before.data, before.status_code, list(before.headers)) == (after.data, after.status_code, list(after.headers)) diff --git a/tests/test_scim_provisioning_worker.mjs b/tests/test_scim_provisioning_worker.mjs new file mode 100644 index 00000000..7073e786 --- /dev/null +++ b/tests/test_scim_provisioning_worker.mjs @@ -0,0 +1,208 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { handleScimStateRequest } from "../worker/scim-d1.mjs"; +import { USER_FIELDS } from "../worker/control-users-d1.mjs"; +import { RevisionConsumer } from "../worker/config-revision.mjs"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; + +const schema = name => readFileSync(new URL(`../intelligence-migrations/${name}`, import.meta.url), "utf8"); +const userSchema = "urn:ietf:params:scim:schemas:core:2.0:User"; +const groupSchema = "urn:ietf:params:scim:schemas:core:2.0:Group"; +const id = n => String(n).padStart(32, "0"); +function resource(n = 1, changes = {}, kind = "Users") { + return { schemas: [kind === "Users" ? userSchema : groupSchema], id: id(n), externalId: `subject-${n}`, + ...(kind === "Users" ? { userName: `user-${n}`, active: true } : { displayName: `team-${n}`, members: [] }), + meta: { resourceType: kind === "Users" ? "User" : "Group", version: 'W/"1"', + created: "2026-10-09T00:00:00Z", lastModified: "2026-10-09T00:00:00Z", location: `/scim/v2/${kind}/${id(n)}` }, ...changes }; +} +function account(n = 1, changes = {}) { + return { ...Object.fromEntries(USER_FIELDS.map(key => [key, null])), username: `user-${n}`, + api_key_hash: "synthetic-hash", api_key_prefix: "mllm_synthetic", is_admin: 0, + scopes: "chat,models", created_at: "2026-10-09T00:00:00Z", ...changes }; +} +function database(t, migrated = true) { + const sql = new DatabaseSync(":memory:"); + t.after(() => sql.close()); + for (const file of ["0003_control_users.sql", "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql", "0017_control_revisions.sql"]) + sql.exec(schema(file)); + sql.exec("CREATE TABLE retained_usage (username TEXT, amount INTEGER); INSERT INTO retained_usage VALUES ('user-1', 7)"); + sql.exec("CREATE TABLE fake_teams (org TEXT, id TEXT, name TEXT, active INTEGER, PRIMARY KEY(org,id))"); + sql.exec("CREATE TABLE fake_principals (org TEXT, username TEXT UNIQUE)"); + if (migrated) sql.exec(schema("0036_scim_provisioning.sql")); + const db = { prepare(query) { + let args = []; + const statement = { bind(...values) { args = values; return statement; }, + first() { return sql.prepare(query).get(...args) ?? null; }, + all() { const p = sql.prepare(query); return p.columns().length ? { results: p.all(...args), meta: { changes: 0 } } + : { results: [], meta: { changes: Number(p.run(...args).changes) } }; } }; + return statement; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { const result = statements.map(s => s.all()); sql.exec("COMMIT"); return result; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const env = { SCIM_ENABLED: "true", CONFIG_REVISION_SYNC_ENABLED: "true", INTELLIGENCE_DB: db }; + const collaborators = { principalStatements: ({ org_id, resource: row }) => [ + db.prepare("INSERT INTO fake_principals VALUES (?,?)").bind(org_id, row.userName)], + teamStatements: ({ org_id, resource: row, deactivated }) => [ + db.prepare("INSERT INTO fake_teams VALUES (?,?,?,?) ON CONFLICT(org,id) DO UPDATE SET name=excluded.name,active=excluded.active") + .bind(org_id, row.id, row.displayName, Number(!deactivated))] }; + const send = async (body, extras = {}) => { + const response = await handleScimStateRequest(new Request("http://intelligence.internal/v1/managed-state/scim", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, org_id: "org:a", ...body }), + }), env, { ...collaborators, ...extras }); + return { status: response.status, ...await response.json() }; + }; + const put = (row, expected = 0, accountRow = null, kind = "Users", extras) => send({ operation: "put", kind, + resource: row, expected, account: accountRow, token_digest: "a".repeat(64), deactivate: false }, extras); + return { sql, db, env, send, put }; +} + +test("atomic account, identity, externalId replay, scoped uniqueness and retained usage", async t => { + const { sql, send, put } = database(t); + assert.equal((await put(resource(), 0, account())).status, 200); + const duplicate = await put(resource(2, { externalId: "subject-1" }), 0, account(2)); + assert.equal(duplicate.resource.id, id(1)); + assert.equal(duplicate.created, false); + assert.equal((await put(resource(2, { userName: "user-1" }), 0, account(2, { username: "user-1" }))).status, 409); + assert.equal((await send({ operation: "get", kind: "Users", id: id(1), org_id: "org:b" })).resource, null); + assert.equal(sql.prepare("SELECT count(*) n FROM control_users").get().n, 1); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_audit").get().n, 1); + assert.equal(sql.prepare("SELECT org FROM fake_principals").get().org, "org:a"); + assert.equal(sql.prepare("SELECT amount FROM retained_usage").get().amount, 7); + for (const changes of [{ is_admin: 1 }, { scopes: "chat,admin" }]) + assert.equal((await put(resource(3), 0, account(3, changes))).status, 400); +}); + +test("CAS concurrent updates roll back account, audit, membership and revision changes", async t => { + const { sql, put } = database(t); + await put(resource(), 0, account()); + const next = resource(1, { active: false, meta: { ...resource().meta, version: 'W/"2"' } }); + const outcomes = await Promise.all([put(next, 1, account(1, { revoked_at: "2026-10-09T01:00:00Z", api_key_prefix: "mllm_changed", api_key_hash: "synthetic-changed-hash" })), + put(next, 1, account(1, { revoked_at: "2026-10-09T01:00:00Z", api_key_prefix: "mllm_other", api_key_hash: "synthetic-other-hash" }))]); + assert.deepEqual(outcomes.map(r => r.status).sort(), [200, 412]); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_audit").get().n, 2); + assert.equal(sql.prepare("SELECT revision FROM control_revisions WHERE domain='key_controls'").get().revision, 2); + assert.equal(sql.prepare("SELECT api_key_prefix FROM control_users").get().api_key_prefix, "mllm_changed"); +}); + +test("deactivation revision invalidates runtime within security TTL", async t => { + const { db, env, put, sql } = database(t); + let now = 0, refreshed = []; + const consumer = new RevisionConsumer(env, { clock: () => now, jitter: () => 1, + refreshers: Object.fromEntries(["model_overrides", "key_controls", "model_grants"].map(name => [name, () => { + refreshed.push(name); return sql.prepare("SELECT revoked_at FROM control_users").get(); }])) }); + await put(resource(), 0, account()); + await consumer.tick(); + assert.equal(consumer.requireFreshSecurity(), null); + const next = resource(1, { active: false, meta: { ...resource().meta, version: 'W/"2"' } }); + await put(next, 1, account(1, { revoked_at: "2026-10-09T01:00:00Z", api_key_prefix: "mllm_changed", api_key_hash: "synthetic-changed-hash" })); + refreshed = []; now = 5; + await consumer.tick(); + assert.ok(refreshed.includes("key_controls") && refreshed.includes("model_grants")); + assert.equal(consumer.requireFreshSecurity(), null); + db.prepare = () => { throw new Error("synthetic outage"); }; + now = 10.01; await consumer.tick(); + assert.equal(consumer.requireFreshSecurity().status, 503); +}); + +test("teams and membership stay tenant bound; collaborator failure rolls back", async t => { + const { sql, put, send } = database(t); + await put(resource(), 0, account()); + const group = resource(10, { members: [{ value: id(1) }] }, "Groups"); + assert.equal((await put(group, 0, null, "Groups")).status, 200); + assert.equal(sql.prepare("SELECT name FROM fake_teams").get().name, "team-10"); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_group_members").get().n, 1); + const foreign = resource(11, { members: [{ value: id(999) }] }, "Groups"); + assert.equal((await put(foreign, 0, null, "Groups")).status, 400); + const failed = await put(resource(12, {}, "Groups"), 0, null, "Groups", { teamStatements: () => { + throw new Error("private details"); } }); + assert.equal(failed.status, 503); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_resources WHERE kind='Groups'").get().n, 1); + assert.equal((await put(resource(13, {}, "Groups"), 0, null, "Groups", { teamStatements: undefined })).status, 503); + const deactivated = { ...group, members: [], meta: { ...group.meta, version: 'W/"2"' } }; + assert.equal((await send({ operation: "put", kind: "Groups", resource: deactivated, expected: 1, + account: null, token_digest: null, deactivate: true })).status, 200); + assert.equal(sql.prepare("SELECT deactivated FROM scim_resources WHERE kind='Groups'").get().deactivated, 1); + assert.equal(sql.prepare("SELECT active FROM fake_teams").get().active, 0); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_group_members").get().n, 0); + const outcomes = await Promise.all([put({ ...group, displayName: "winning", meta: { ...group.meta, version: 'W/"3"' } }, 2, null, "Groups"), + put({ ...group, displayName: "losing", meta: { ...group.meta, version: 'W/"3"' } }, 2, null, "Groups")]); + assert.deepEqual(outcomes.map(result => result.status).sort(), [200, 412]); + assert.equal(sql.prepare("SELECT name FROM fake_teams").get().name, "winning"); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_group_members").get().n, 1); +}); + +test("missing schema, disabled flag, private target and invalid bodies fail before changes", async t => { + const { sql, env, put } = database(t, false); + assert.equal((await put(resource(), 0, account())).status, 503); + assert.equal(sql.prepare("SELECT count(*) n FROM control_users").get().n, 0); + const req = (url = "http://intelligence.internal/v1/managed-state/scim", body = "{}") => new Request(url, { + method: "POST", headers: { "content-type": "application/json" }, body }); + for (const flag of [undefined, "", "false", "invalid-private"]) + assert.equal((await handleScimStateRequest(req(), { SCIM_ENABLED: flag, INTELLIGENCE_DB: { + prepare() { assert.fail("off touched storage"); } } })).status, 404); + assert.equal((await handleScimStateRequest(req("https://external.test/v1/managed-state/scim"), env)).status, 404); + assert.equal((await handleScimStateRequest(req(undefined, "invalid"), env)).status, 400); + assert.equal((await handleScimStateRequest(req(undefined, "a".repeat(140000)), env)).status, 400); +}); + +test("migration is additive and repeats without destroying existing rows; list count is capped", async t => { + const { sql, send, put } = database(t); + await put(resource(), 0, account()); + sql.exec(schema("0036_scim_provisioning.sql")); + const listed = await send({ operation: "list", kind: "Users", attribute: "userName", value: "user-1", start: 1, count: 100 }); + assert.equal(listed.total, 1); + assert.equal(listed.resources.length, 1); + assert.equal((await send({ operation: "list", kind: "Users", attribute: "roles", value: "admin", start: 1, count: 100 })).status, 400); + assert.equal((await send({ operation: "list", kind: "Users", attribute: null, value: null, start: 1, count: 101 })).status, 400); +}); + +test("mutations require atomic principal authority and fail closed on transaction errors", async t => { + const { sql, put, env, send } = database(t); + assert.equal((await put(resource(), 0, account(), "Users", { principalStatements: undefined })).status, 503); + assert.equal(sql.prepare("SELECT count(*) n FROM control_users").get().n, 0); + assert.equal((await put(resource(), 0, account(), "Users", { principalStatements: () => [ + env.INTELLIGENCE_DB.prepare("INSERT INTO nonexistent_authority VALUES (1)")] })).status, 503); + assert.equal(sql.prepare("SELECT count(*) n FROM scim_resources").get().n, 0); + assert.equal(sql.prepare("SELECT count(*) n FROM control_revisions").get().n, 0); + env.CONFIG_REVISION_SYNC_ENABLED = "false"; + assert.equal((await send({ operation: "probe" })).status, 400); + const response = await handleScimStateRequest(new Request("http://intelligence.internal/v1/managed-state/scim", { + method: "POST", headers: { "content-type": "application/json" }, body: '{"version":1,"operation":"probe"}', + }), env); + assert.equal(response.status, 503); +}); + +test("missing owned columns and security revision schema precede account mutations", async t => { + const { sql, put } = database(t, false); + sql.exec("CREATE TABLE scim_resources (org_id TEXT); INSERT INTO scim_resources VALUES ('retained')"); + sql.exec(schema("0036_scim_provisioning.sql").split("CREATE UNIQUE INDEX", 1)[0]); + assert.equal((await put(resource(), 0, account())).status, 503); + assert.equal(sql.prepare("SELECT org_id FROM scim_resources").get().org_id, "retained"); + assert.equal(sql.prepare("SELECT count(*) n FROM control_users").get().n, 0); + const second = database(t); + second.sql.exec("DROP TABLE control_revisions"); + assert.equal((await second.put(resource(), 0, account())).status, 503); + assert.equal(second.sql.prepare("SELECT count(*) n FROM control_users").get().n, 0); +}); + +test("public SCIM routes forward body, bearer and If-Match unchanged to Container", async () => { + const { default: worker } = await loadWorkerModule(); + const captured = []; + const env = { MULTILLM_PROXY_CONTAINER: { getByName() { return { async fetch(request) { + captured.push({ path: new URL(request.url).pathname, body: await request.text(), headers: request.headers }); + return new Response("{}", { headers: { "content-type": "application/scim+json" } }); + } }; } } }; + const body = '{"Operations":[{"op":"replace","path":"active","value":false}]}'; + const response = await worker.fetch(new Request("https://gateway.test/scim/v2/Users/test", { method: "PATCH", body, + headers: { "Authorization": "Bearer synthetic-scim-token", "Content-Type": "application/scim+json", "If-Match": 'W/"1"' } }), env); + assert.equal(response.status, 200); + assert.equal(captured.length, 1); + assert.equal(captured[0].path, "/scim/v2/Users/test"); + assert.equal(captured[0].body, body); + assert.equal(captured[0].headers.get("Authorization"), "Bearer synthetic-scim-token"); + assert.equal(captured[0].headers.get("If-Match"), 'W/"1"'); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..c4beb091 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "SCIM_ENABLED", + "SCIM_TRUST_CONFIG_JSON", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..5bf5466c 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "scim_resources", "scim_group_members", "scim_token_digests", "scim_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/scim-d1.mjs b/worker/scim-d1.mjs new file mode 100644 index 00000000..b1379d7f --- /dev/null +++ b/worker/scim-d1.mjs @@ -0,0 +1,248 @@ +/** Atomic SCIM identity, account, team and security-revision storage. */ +import { boundedBody, USER_FIELDS, validUser } from "./control-users-d1.mjs"; +import { commitRevision, requireRevisionSchema, revisionSyncSettings } from "./config-revision.mjs"; + +const PATH = "/v1/managed-state/scim"; +const ID = /^[a-f0-9]{32}$/; +const ORG = /^[A-Za-z0-9_:.\-]{1,128}$/; +const HASH = /^[a-f0-9]{64}$/; +const VERSION = /^W\/"([1-9][0-9]{0,15})"$/; +const USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User"; +const GROUP_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:Group"; +const TABLE_COLUMNS = { + scim_resources: "org_id,kind,id,external_id,user_name,display_name,deactivated,revision,document", + scim_group_members: "org_id,group_id,user_id", + scim_token_digests: "org_id,digest_sha256,updated_at", + scim_audit: "sequence,org_id,kind,resource_id,revision,operation,at", +}; +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const exact = (value, names) => object(value) && Object.keys(value).length === names.length && names.every(key => Object.hasOwn(value, key)); +const string = (value, max = 1024) => typeof value === "string" && value.length > 0 && value.length <= max && !/[\x00-\x1f\x7f]/.test(value); +const version = resource => Number(VERSION.exec(resource.meta.version)?.[1]); +let warned = false; + +export function scimEnabled(env) { + const raw = env.SCIM_ENABLED ?? ""; + const flag = typeof raw === "string" ? raw.trim().toLowerCase() : "invalid"; + if (["", "false", "0", "no", "off"].includes(flag)) return false; + if (["true", "1", "yes", "on"].includes(flag)) return true; + if (!warned) { warned = true; console.warn("Invalid SCIM_ENABLED; SCIM provisioning disabled"); } + return false; +} + +const reply = (value, status = 200) => Response.json(status === 200 ? { version: 1, ...value } + : { version: 1, error: { code: value, message: "SCIM storage could not accept the operation" } }, + { status, headers: { "cache-control": "no-store" } }); + +function validResource(resource, kind, expected) { + if (!object(resource) || !ID.test(resource.id ?? "") || !object(resource.meta)) return false; + const attrs = kind === "Users" ? ["userName", "active", "displayName", "externalId", "name", "emails"] + : ["displayName", "externalId", "members"]; + if (Object.keys(resource).some(key => !["schemas", "id", "meta", ...attrs].includes(key)) + || JSON.stringify(resource.schemas) !== JSON.stringify([kind === "Users" ? USER_SCHEMA : GROUP_SCHEMA]) + || Object.keys(resource.meta).some(key => !["resourceType", "version", "created", "lastModified", "location"].includes(key)) + || resource.meta.resourceType !== (kind === "Users" ? "User" : "Group") + || !VERSION.test(resource.meta.version ?? "") || version(resource) !== expected + 1 + || resource.meta.location !== `/scim/v2/${kind}/${resource.id}` + || !["created", "lastModified"].every(key => string(resource.meta[key], 64) && Number.isFinite(Date.parse(resource.meta[key]))) + || ["displayName", "externalId"].some(key => key in resource && !string(resource[key]))) return false; + if (kind === "Groups") return string(resource.displayName) && Array.isArray(resource.members) && resource.members.length <= 1000 + && new Set(resource.members.map(m => m?.value)).size === resource.members.length + && resource.members.every(m => object(m) && ID.test(m.value ?? "") + && Object.keys(m).every(key => ["value", "display", "$ref"].includes(key)) + && ["display", "$ref"].every(key => !(key in m) || string(m[key]))); + if (!string(resource.userName, 128) || resource.userName.trim() !== resource.userName || typeof resource.active !== "boolean") return false; + if ("name" in resource && (!object(resource.name) || Object.keys(resource.name).some(key => + !["formatted", "familyName", "givenName", "middleName", "honorificPrefix", "honorificSuffix"].includes(key)) + || !Object.values(resource.name).every(value => string(value)))) return false; + if ("emails" in resource && (!Array.isArray(resource.emails) || resource.emails.length > 100 + || resource.emails.filter(e => e?.primary === true).length > 1 || !resource.emails.every(e => object(e) && string(e.value) + && Object.keys(e).every(key => ["value", "type", "display", "primary"].includes(key)) + && (!Object.hasOwn(e, "primary") || typeof e.primary === "boolean") + && ["type", "display"].every(key => !(key in e) || string(e[key]))))) return false; + return true; +} + +function validBody(body) { + if (!object(body) || body.version !== 1) return false; + if (body.operation === "probe") return exact(body, ["version", "operation"]); + if (body.operation === "account") return exact(body, ["version", "operation", "username"]) && string(body.username, 128); + if (!ORG.test(body.org_id ?? "") || !["Users", "Groups"].includes(body.kind)) return false; + const base = ["version", "operation", "org_id", "kind"]; + if (body.operation === "get") return exact(body, [...base, "id"]) && ID.test(body.id ?? ""); + if (body.operation === "list") return exact(body, [...base, "attribute", "value", "start", "count"]) + && Number.isInteger(body.start) && body.start >= 1 && body.start <= 999999999 + && Number.isInteger(body.count) && body.count >= 0 && body.count <= 100 + && (body.attribute === null && body.value === null || ["userName", "externalId", "displayName"].includes(body.attribute) && string(body.value)); + return body.operation === "put" && exact(body, [...base, "resource", "expected", "account", "token_digest", "deactivate"]) + && Number.isSafeInteger(body.expected) && body.expected >= 0 && body.expected < 9007199254740991 + && validResource(body.resource, body.kind, body.expected) + && (body.token_digest === null || HASH.test(body.token_digest ?? "")) + && typeof body.deactivate === "boolean" && (!body.deactivate || body.kind === "Groups" && body.resource.members.length === 0) + && (body.account === null || body.kind === "Users" && validUser(body.account) && body.account.is_admin === 0 + && (body.expected !== 0 || body.account.scopes === "chat,models") && body.account.username === body.resource.userName + && (body.resource.active ? body.account.revoked_at === null : body.account.revoked_at !== null)); +} + +async function requireSchema(db) { + if (!db) throw new Error("Storage unavailable"); + // Probe every owned table and all account columns before preparing any account change. + for (const [table, columns] of Object.entries(TABLE_COLUMNS)) await db.prepare(`SELECT ${columns} FROM ${table} LIMIT 0`).all(); + await db.prepare(`SELECT ${USER_FIELDS.join(",")} FROM control_users LIMIT 0`).all(); + await requireRevisionSchema(db); +} + +async function get(db, org, kind, id) { + const row = await db.prepare("SELECT document FROM scim_resources WHERE org_id=? AND kind=? AND id=?") + .bind(org, kind, id).first(); + return row ? JSON.parse(row.document) : null; +} + +async function list(db, body) { + const column = { userName: "user_name", externalId: "external_id", displayName: "display_name" }[body.attribute]; + const where = `org_id=? AND kind=?${column ? ` AND ${column}=?` : ""}`; + const args = [body.org_id, body.kind, ...(column ? [body.value] : [])]; + const count = await db.prepare(`SELECT count(*) AS total FROM scim_resources WHERE ${where}`).bind(...args).first(); + const rows = await db.prepare(`SELECT document FROM scim_resources WHERE ${where} ORDER BY id LIMIT ? OFFSET ?`) + .bind(...args, body.count, body.start - 1).all(); + return reply({ resources: rows.results.map(row => JSON.parse(row.document)), total: count.total }); +} + +function casGuard(db, body) { + return db.prepare(`INSERT INTO scim_resources (org_id,kind,id,revision,document) + SELECT ?,?,?,-1,'' WHERE changes() != 1`).bind(body.org_id, body.kind, body.resource.id); +} + +function resourceStatements(db, body) { + const { org_id: org, kind, resource: row, expected } = body; + const values = [row.externalId ?? null, row.userName ?? null, row.displayName ?? null, expected + 1, JSON.stringify(row), Number(body.deactivate)]; + const statement = expected === 0 + ? db.prepare("INSERT INTO scim_resources (external_id,user_name,display_name,revision,document,deactivated,org_id,kind,id) VALUES (?,?,?,?,?,?,?,?,?)") + .bind(...values, org, kind, row.id) + : db.prepare("UPDATE scim_resources SET external_id=?,user_name=?,display_name=?,revision=?,document=?,deactivated=? WHERE org_id=? AND kind=? AND id=? AND revision=?") + .bind(...values, org, kind, row.id, expected); + return [statement, casGuard(db, body)]; +} + +function accountStatements(db, body, priorAccount) { + if (!body.account) return []; + const { account, expected } = body; + if (expected === 0) return [db.prepare(`INSERT INTO control_users (${USER_FIELDS.join(",")}) VALUES (${USER_FIELDS.map(() => "?").join(",")})`) + .bind(...USER_FIELDS.map(key => account[key]))]; + return [db.prepare(`UPDATE control_users SET api_key_hash=?,api_key_prefix=?,revoked_at=? + WHERE username=? AND is_admin=0 AND api_key_hash=? AND api_key_prefix=? AND revoked_at IS ?`) + .bind(account.api_key_hash, account.api_key_prefix, account.revoked_at, account.username, + priorAccount.api_key_hash, priorAccount.api_key_prefix, priorAccount.revoked_at), casGuard(db, body)]; +} + +async function prepareAccount(db, body, old) { + if (body.kind !== "Users") return null; + const existing = await db.prepare(`SELECT ${USER_FIELDS.join(",")} FROM control_users WHERE username=?`).bind(body.resource.userName).first(); + if (body.expected === 0) { + if (!body.account) throw Object.assign(new Error(), { status: 400 }); + if (existing) throw Object.assign(new Error(), { status: 409 }); + return null; + } + if (!existing || existing.is_admin !== 0 || body.resource.userName !== old.userName) throw Object.assign(new Error(), { status: 400 }); + const changed = body.resource.active !== old.active; + if (changed !== (body.account !== null)) throw Object.assign(new Error(), { status: 400 }); + if (body.account) { + for (const field of USER_FIELDS.filter(key => !["api_key_hash", "api_key_prefix", "revoked_at"].includes(key))) { + if (body.account[field] !== existing[field]) throw Object.assign(new Error(), { status: 412 }); + } + if (!body.resource.active && (body.account.api_key_hash === existing.api_key_hash || body.account.api_key_prefix === existing.api_key_prefix)) + throw Object.assign(new Error(), { status: 400 }); + if (body.resource.active && (body.account.api_key_hash !== existing.api_key_hash || body.account.api_key_prefix !== existing.api_key_prefix)) + throw Object.assign(new Error(), { status: 400 }); + } + return existing; +} + +function collaboratorStatements(callback, data) { + if (typeof callback !== "function") throw new Error("Authority unavailable"); + const result = callback(data); + if (!Array.isArray(result)) throw new Error("Atomic authority statements required"); + return result; +} + +async function put(db, body, collaborators) { + const { org_id: org, kind, resource: row, expected } = body; + const old = await get(db, org, kind, row.id); + if (expected === 0 && row.externalId) { + const existing = await db.prepare("SELECT document FROM scim_resources WHERE org_id=? AND kind=? AND external_id=?") + .bind(org, kind, row.externalId).first(); + if (existing) return reply({ resource: JSON.parse(existing.document), created: false }); + } + if (expected !== 0 && (!old || version(old) !== expected)) return reply("scim_version_conflict", 412); + if (kind === "Users") { + const names = new Set([String(collaborators.adminUsername ?? "admin"), ...(collaborators.adminUsernames ?? [])]); + if (names.has(row.userName)) return reply("scim_admin_denied", 400); + } + const priorAccount = await prepareAccount(db, body, old); + const statements = resourceStatements(db, body); + statements.push(...accountStatements(db, body, priorAccount)); + if (kind === "Users" && expected === 0) { + statements.push(...collaboratorStatements(collaborators.principalStatements, { org_id: org, resource: row })); + } + if (kind === "Groups") { + statements.push(...collaboratorStatements(collaborators.teamStatements, { org_id: org, resource: row, prior: old, deactivated: body.deactivate })); + statements.push(db.prepare("DELETE FROM scim_group_members WHERE org_id=? AND group_id=?").bind(org, row.id)); + for (const member of row.members) { + const user = await get(db, org, "Users", member.value); + if (!user) return reply("scim_foreign_member", 400); + statements.push(db.prepare(`INSERT INTO scim_group_members (org_id,group_id,user_id) + SELECT ?,?,id FROM scim_resources WHERE org_id=? AND kind='Users' AND id=?`) + .bind(org, row.id, org, member.value), casGuard(db, body)); + } + } + const now = new Date().toISOString(); + if (body.token_digest !== null) statements.push(db.prepare(`INSERT INTO scim_token_digests VALUES (?,?,?) + ON CONFLICT(org_id) DO UPDATE SET digest_sha256=excluded.digest_sha256,updated_at=excluded.updated_at`) + .bind(org, body.token_digest, now)); + statements.push(db.prepare("INSERT INTO scim_audit (org_id,kind,resource_id,revision,operation,at) VALUES (?,?,?,?,?,?)") + .bind(org, kind, row.id, expected + 1, row.active === false || body.deactivate ? "deactivate" : expected === 0 ? "create" : "update", now)); + try { + if (!await commitRevision(db, ["key_controls", "model_grants"], statements)) return reply("scim_revision_conflict", 412); + } catch (error) { + const message = String(error?.message ?? error); + if (message.includes("scim_cas_confirmed")) return reply("scim_version_conflict", 412); + if (/UNIQUE constraint/i.test(message)) { + if (expected === 0 && row.externalId) { + const existing = await db.prepare("SELECT document FROM scim_resources WHERE org_id=? AND kind=? AND external_id=?") + .bind(org, kind, row.externalId).first(); + if (existing) return reply({ resource: JSON.parse(existing.document), created: false }); + } + return reply("scim_uniqueness", 409); + } + throw error; + } + return reply({ resource: row, created: expected === 0 }); +} + +export async function handleScimStateRequest(request, env, collaborators = {}) { + if (!scimEnabled(env)) return reply("not_found", 404); + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== PATH || url.search || url.hash + || url.username || url.password || request.method !== "POST") return reply("not_found", 404); + let body; + try { + if (!/^application\/json(?:;|$)/i.test(request.headers.get("content-type") ?? "")) throw new Error(); + body = JSON.parse(await boundedBody(request, 131072)); + if (!validBody(body)) throw new Error(); + } catch { return reply("scim_invalid_request", 400); } + try { + if (!revisionSyncSettings(env).enabled) return reply("scim_security_sync_required", 503); + const db = env.INTELLIGENCE_DB; + await requireSchema(db); + if (body.operation === "probe") return reply({ ready: true }); + if (body.operation === "get") return reply({ resource: await get(db, body.org_id, body.kind, body.id) }); + if (body.operation === "list") return await list(db, body); + if (body.operation === "account") return reply({ account: await db.prepare(`SELECT ${USER_FIELDS.join(",")} FROM control_users WHERE username=?`) + .bind(body.username).first() }); + return await put(db, body, { ...collaborators, adminUsername: String(env.ADMIN_USERNAME || "admin").trim(), + adminUsernames: String(env.ADMIN_USERNAMES || "").split(",").map(name => name.trim()) }); + } catch (error) { + const status = [400, 409, 412].includes(error?.status) ? error.status : 503; + return reply(status === 503 ? "scim_storage_unavailable" : "scim_operation_rejected", status); + } +} From 4d362a75c0f1106f16234dd069fde2680a073aaf Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:35:04 +0530 Subject: [PATCH 4/9] feat(credits): add an opt-in gateway credits ledger CREDITS_ENABLED (off by default) keeps an append-only integer micro-USD ledger per owner: credit, reserve, commit, release, adjust and compensate entries, each idempotent by operation ID and written with a compare-and-swap on the owner's balance revision. Corrections append compensation entries and never rewrite history. Reservations and negative adjustments may not make available credit negative. A commit always records the actual cost and closes its hold, so an overrun leaves available credit negative and new reservations are refused until credits cover it. An unknown cost keeps the hold. GET /v1/credits returns the owner's balance and paged entries, and POST /admin/credits/adjust needs an administrator session and If-Match. The ledger implements the enterprise CreditAuthority; connecting it to request reservations comes with integration (migration 0037). --- docs/credits-ledger.md | 157 +++++++ .../0037_credits_ledger.sql | 33 ++ package.json | 2 +- routes/credits.py | 94 ++++ services/credits_ledger.py | 401 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 6 + tests/test_credits_ledger.py | 398 +++++++++++++++++ tests/test_credits_ledger_worker.mjs | 226 ++++++++++ worker/container-env.mjs | 2 + worker/credits-d1.mjs | 273 ++++++++++++ worker/d1-schema.mjs | 2 +- 11 files changed, 1592 insertions(+), 2 deletions(-) create mode 100644 docs/credits-ledger.md create mode 100644 intelligence-migrations/0037_credits_ledger.sql create mode 100644 routes/credits.py create mode 100644 services/credits_ledger.py create mode 100644 tests/test_credits_ledger.py create mode 100644 tests/test_credits_ledger_worker.mjs create mode 100644 worker/credits-d1.mjs diff --git a/docs/credits-ledger.md b/docs/credits-ledger.md new file mode 100644 index 00000000..5ace9fb7 --- /dev/null +++ b/docs/credits-ledger.md @@ -0,0 +1,157 @@ +# Gateway credits + +Credits are disabled unless `CREDITS_ENABLED=true`. Empty values use the defaults. +`CREDITS_CURRENCY` defaults to USD; other currencies disable credits. Invalid +settings log one warning without their values. Disabled routes return 404. + +Apply the additive credits migration to the chosen database before enabling the +feature. It creates `credits_balances`, `credits_entries` and `credits_audit`. +Existing usage records are retained. The runtime does not apply migrations. +Missing tables or required columns return JSON 503 with `credits_unavailable`; +credit admission then stops before provider dispatch. Storage outages have the +same result, without retrying writes or switching to another balance authority. +The local store uses the existing `USAGE_DB_PATH`; the Container uses the private +D1 transport when `INTELLIGENCE_STORAGE_BACKEND=d1`. + +Amounts are integer micro-USD, bounded to ±1,000,000,000,000,000. One USD is +1,000,000 micro-USD. Fractional amounts, booleans and overflow are rejected, +without rounding or conversion from usage estimates. Revisions have the same +nonnegative bound. No account receives automatic credit, bonus or top-up. + +The ledger stores `credit`, `reserve`, `commit`, `release`, `adjust` and +`compensate` events. Only the materialized balance row changes. Balance is the +sum of credits and adjustments, minus committed costs, plus compensations. +Held credit is the sum of open estimates; available credit is balance minus +held. Every write checks the balance revision. New reservations and negative +adjustments must not make available credit negative. Each owner has its own +operation ID namespace. Repeating an identical +operation, including its original revision, returns its original entry. Reusing +the ID with different content returns 409 `credits_conflict`. + +Use a distinct `scoped_id` for each provider attempt, including hedges, shadows +and batch items, and a distinct operation ID for each transition. Reserving an +estimate holds credit. Committing the actual integer cost removes that attempt's +whole hold and debits only the actual cost. The commit entry's `held_delta` +records the released hold atomically. An explicit release removes a hold without +a charge; its amount must match the original estimate. Terminal attempts cannot +reopen. + +A commit records the actual cost even when it exceeds the hold and available +credit, and closes only its own hold. Balance and available credit can become +negative within the integer bounds. New reservations are refused while available +credit is zero or negative, until credits or positive adjustments cover the +shortfall and leave available credit for admission. + +Unknown reconciliation appends a linked reserve marker with zero balance and +hold deltas. Its zero amount is a marker, not a zero-cost settlement. The original +hold remains indefinitely; there is no automatic expiry. A later known cost can +settle it. A compensation references an original credit, adjustment or commit +operation and applies the exact inverse balance delta once. It never edits the +original entry, reopens a hold or modifies a usage receipt. + +Gateway prices are an explicit operator tariff, separate from provider usage +estimates and invoices. `register_credit_authority(store, tariff=...)` and +`registerCreditAuthority(db, {tariff, env})` implement the enterprise +`CreditAuthority` contract. The tariff callback receives `(operation, phase)`; +it must verify the operation's integer amount against the reviewed tariff and +return that tariff's integer revision. Return `None`/`null` for unpriced admission +or unknown reconciliation. Unpriced reserve or commit raises `AuthorityDenied` +with `credits_unpriced`; insufficient funds raise `credits_insufficient`. +Registration performs no storage writes. The authority also checks the feature +flag before evaluating prices or touching storage. + +For every authority transition, `AuthorityOperation.revision` must be the owner's +current balance revision. On 412 `credits_revision_mismatch`, re-read that revision +and retry with the same operation ID and unchanged cost and attempt fields, using +a bounded retry limit. A rejected revision writes nothing, and operation IDs make +retries idempotent. After a successful write, replay the original revision to +retrieve its entry; reusing the ID with changed content is a conflict. + +`GET /v1/credits` requires a gateway key with `models` scope. It reads only the +authenticated owner; query parameters cannot choose another owner. Verified +tenant context produces a stable organisation/team/principal namespace; +legacy principals keep their existing namespace. The response is: + +```text +{currency: "USD", balance_microusd: 100, held_microusd: 60, + available_microusd: 40, entries: [...], next_cursor: null} +``` + +Use `?limit=20&cursor=0` for pagination. The limit is 1–100 (default 100); cursors +are nonnegative revision strings. Follow `next_cursor` until it is null. Pages +are ordered by increasing ledger revision. All responses use `Cache-Control: +no-store`. Reads and writes include an `ETag` containing the balance revision. +Entries contain operation IDs, integer deltas, attempt/reference IDs and tariff +revisions. They omit actor and adjustment reason. No prompts, completions, +provider credentials or payment details are collected. + +`GET /admin/credits/` requires an administrator dashboard session and +includes the current revision. URL-encode tenant owner IDs. +`POST /admin/credits/adjust` requires the same session, a CSRF token and an +`If-Match` header copied from the owner's current ETag. For example: + +```text +If-Match: "0" +X-CSRFToken: +{"owner":"alice","amount_microusd":1000000, + "operation_id":"allocation-1","reason":"Reviewed allocation"} +``` + +Missing `If-Match` returns 428 `credits_revision_required`; a stale revision +returns 412 `credits_revision_mismatch`. Negative adjustments are allowed only +within available credit. Reasons must be 1–512 printable ASCII characters. +Audit rows atomically record owner, operation ID, revision, actor and kind, with +no reason or request content. Treat reason text as administrative metadata and +keep user content out of it. Ledger and audit history are retained; this feature +provides no deletion, automatic payment, currency conversion or invoice service. + +Application integration uses `register_credits_routes(app, csrf)` in +`app.py:create_app`. Bind verified context as `g.tenant_context`. Supply the +registered credit authority to `services/enterprise_contract.py: +register_enterprise_adapters`, preserving the other authority collaborators. +Reservation integration calls reserve from `services/budget_service.py: +BudgetService._reserve_durable` / `services/request_accounting.py:begin`, commit +from `BudgetService.complete` / `services/request_accounting.py:_record_budget`, +and reconcile from `services/reservation_store.py:reconcile`. Use independent +attempt identities and the current balance revision. The private dispatcher call site is +`worker/managed-state-dispatch.mjs:handleManagedStateRequest`, mapped to +`handleCreditsRequest` for `/v1/managed-state/credits`. The Container transport +uses the `credits` domain, allowlisted as `/v1/managed-state/credits` in +`services/intelligence_d1_store.py:_ENDPOINTS` on the private internal origin. +For native requests, bind the authority at the reserve/settle/reconcile call +sites in `worker/reservations-d1.mjs:createReservationLifecycle` through the +fixed registration in `worker/gateway-extensions.mjs`. These call sites must +classify unknown costs rather than translating them to zero. `CreditDenied` +extends `AuthorityDenied` with bounded `code` and `status`; Flask route +registration installs its JSON error handler. Native request integration uses +the denial's `response()` method through `generationErrorResponse` so schema +failures remain JSON 503 denials. + +Usage receipts may include an optional ledger operation ID and revision as +provenance. Credit admission and settlement do not require receipt storage, +and receipt errors must never trigger a second charge. Local tests exercise +SQLite and a SQLite-backed D1 contract fake; they do not validate deployed D1, +live provider metering or an operator's tariff. + +Shared integer and idempotency vectors: + +```json +[ + {"invalid_amount": true}, + {"invalid_amount": 0.5}, + {"invalid_amount": 1000000000000001}, + {"invalid_amount": -1000000000000001}, + {"request": {"owner":"alice","kind":"credit","amount_microusd":100,"operation_id":"fund","revision":0},"revision":1}, + {"request": {"owner":"alice","kind":"credit","amount_microusd":100,"operation_id":"fund","revision":0},"revision":1}, + {"request": {"owner":"alice","kind":"credit","amount_microusd":101,"operation_id":"fund","revision":0},"error":"credits_conflict"}, + {"request": {"owner":"alice","kind":"reserve","amount_microusd":60,"operation_id":"hold","revision":1,"scoped_id":"attempt","tariff_revision":1},"revision":2}, + {"request": {"owner":"alice","kind":"reserve","amount_microusd":50,"operation_id":"second","revision":2,"scoped_id":"other","tariff_revision":1},"error":"credits_insufficient"}, + {"request": {"owner":"alice","kind":"commit","amount_microusd":25,"operation_id":"charge","revision":2,"scoped_id":"attempt","tariff_revision":1},"revision":3}, + {"request":{"owner":"overrun","kind":"credit","amount_microusd":100,"operation_id":"fund","revision":0},"revision":1}, + {"request":{"owner":"overrun","kind":"reserve","amount_microusd":60,"operation_id":"hold","revision":1,"scoped_id":"attempt","tariff_revision":1},"revision":2}, + {"request":{"owner":"overrun","kind":"commit","amount_microusd":150,"operation_id":"charge","revision":2,"scoped_id":"attempt","tariff_revision":1},"revision":3,"summary":{"balance_microusd":-50,"held_microusd":0,"available_microusd":-50}}, + {"request":{"owner":"overrun","kind":"reserve","amount_microusd":1,"operation_id":"denied","revision":3,"scoped_id":"next","tariff_revision":1},"error":"credits_insufficient"}, + {"request":{"owner":"overrun","kind":"credit","amount_microusd":100,"operation_id":"refill","revision":3},"revision":4,"summary":{"balance_microusd":50,"held_microusd":0,"available_microusd":50}}, + {"request":{"owner":"overrun","kind":"reserve","amount_microusd":1,"operation_id":"next","revision":4,"scoped_id":"next","tariff_revision":1},"revision":5,"summary":{"balance_microusd":50,"held_microusd":1,"available_microusd":49}} +] +``` diff --git a/intelligence-migrations/0037_credits_ledger.sql b/intelligence-migrations/0037_credits_ledger.sql new file mode 100644 index 00000000..528b9ed5 --- /dev/null +++ b/intelligence-migrations/0037_credits_ledger.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS credits_balances ( + owner TEXT PRIMARY KEY, + balance_microusd INTEGER NOT NULL DEFAULT 0 CHECK (balance_microusd BETWEEN -1000000000000000 AND 1000000000000000), + held_microusd INTEGER NOT NULL DEFAULT 0 CHECK (held_microusd BETWEEN 0 AND 1000000000000000), + revision INTEGER NOT NULL DEFAULT 0 CHECK (revision BETWEEN 0 AND 1000000000000000) +); +CREATE TABLE IF NOT EXISTS credits_entries ( + owner TEXT NOT NULL, + operation_id TEXT NOT NULL, + revision INTEGER NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('credit','reserve','commit','release','adjust','compensate')), + amount_microusd INTEGER NOT NULL CHECK (amount_microusd BETWEEN -1000000000000000 AND 1000000000000000), + balance_delta INTEGER NOT NULL, + held_delta INTEGER NOT NULL, + scoped_id TEXT, + reference TEXT, + tariff_revision INTEGER, + actor TEXT, + reason TEXT, + document TEXT NOT NULL, + PRIMARY KEY (owner, operation_id), + UNIQUE (owner, revision) +); +CREATE UNIQUE INDEX IF NOT EXISTS credits_compensation_once ON credits_entries(owner, reference) WHERE kind = 'compensate'; +CREATE INDEX IF NOT EXISTS credits_attempt_entries ON credits_entries(owner, scoped_id, revision); +CREATE TABLE IF NOT EXISTS credits_audit ( + owner TEXT NOT NULL, + operation_id TEXT NOT NULL, + revision INTEGER NOT NULL, + actor TEXT NOT NULL, + kind TEXT NOT NULL, + PRIMARY KEY (owner, operation_id) +); diff --git a/package.json b/package.json index c66e4a8f..3e416fbc 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_credits_ledger_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/credits.py b/routes/credits.py new file mode 100644 index 00000000..d4b42852 --- /dev/null +++ b/routes/credits.py @@ -0,0 +1,94 @@ +"""Owner reads and explicit administrator CAS adjustments for gateway credits.""" +import re + +from flask import g, jsonify, request + +from route_helpers import api_authenticate_only, login_required +from routes.config_snapshots import json_errors +from routes.core import require_admin_dashboard_user +from services import credits_ledger as ledger + + +def register_credits_routes(app, csrf, *, store=None): + if app.extensions.get('credits_routes_registered'): + return + app.extensions['credits_routes_registered'] = True + + def disabled_credits(): + if request.path == '/v1/credits' or request.path.startswith('/admin/credits/'): + if not ledger.enabled(): + return failed(ledger.CreditsError('not_found', 404)) + return None + + app.before_request_funcs.setdefault(None, []).insert(0, disabled_credits) + + @app.after_request + def private_credits(result): + if request.path == '/v1/credits' or request.path.startswith('/admin/credits/'): + result.headers['Cache-Control'] = 'no-store' + return result + + def authority(): + if not ledger.enabled(): + raise ledger.CreditsError('not_found', 404) + return store if store is not None else ledger.open_store() + + def failed(error): + return jsonify({'error': {'code': error.code, 'message': 'Credits operation unavailable.'}}), error.status, {'Cache-Control': 'no-store'} + + app.register_error_handler(ledger.CreditDenied, failed) + + def response(value, *, owner_view=False): + result = jsonify({key: item for key, item in value.items() if not owner_view or key != 'revision'}) + result.headers['Cache-Control'] = 'no-store' + if 'revision' in value: + result.headers['ETag'] = '"' + str(value['revision']) + '"' + return result + + def page(owner): + limit = request.args.get('limit', '100') + if not re.fullmatch(r'[0-9]{1,3}', limit): + raise ledger.CreditsError('invalid_credits_request', 400) + return authority().read(owner, cursor=request.args.get('cursor', '0'), limit=int(limit)) + + @app.route('/v1/credits', methods=['GET', 'OPTIONS']) + @csrf.exempt + @api_authenticate_only(required_scope='models') + def owner_credits(): + try: + context = getattr(g, 'tenant_context', None) + user = g.authenticated_user + owner = ledger.context_owner(context) if context is not None else str(user.get('username') or user.get('id') or '') + return response(page(owner), owner_view=True) + except ledger.CreditsError as error: + return failed(error) + + @app.get('/admin/credits/') + @json_errors + @login_required + def admin_credits(owner): + try: + require_admin_dashboard_user() + return response(page(owner)) + except ledger.CreditsError as error: + return failed(error) + + @app.post('/admin/credits/adjust') + @json_errors + @login_required + def adjust_credits(): + try: + actor = require_admin_dashboard_user() + match = request.headers.get('If-Match') + if match is None: + raise ledger.CreditsError('credits_revision_required', 428) + if not re.fullmatch(r'"[0-9]{1,16}"', match): + raise ledger.CreditsError('invalid_credits_request', 400) + body = request.get_json(silent=True) + if not isinstance(body, dict) or set(body) != {'owner', 'amount_microusd', 'operation_id', 'reason'}: + raise ledger.CreditsError('invalid_credits_request', 400) + entry = authority().append(body['owner'], 'adjust', body['amount_microusd'], body['operation_id'], + int(match[1:-1]), reason=body['reason'], actor=str(actor.get('username') or actor.get('id') or '')) + return response(entry) + except ledger.CreditsError as error: + return failed(error) diff --git a/services/credits_ledger.py b/services/credits_ledger.py new file mode 100644 index 00000000..63a85e71 --- /dev/null +++ b/services/credits_ledger.py @@ -0,0 +1,401 @@ +"""Immutable integer credits and revision-bound, fail-closed spending authority.""" +from __future__ import annotations + +import json +import logging +import os +import re +import sqlite3 +import threading +from contextlib import closing +from pathlib import Path +from typing import Mapping + +from services.enterprise_contract import (AuthorityDenied, AuthorityOperation, AuthorityResult, + TenantContext, register_enterprise_adapters) +from services.sqlite_store import storage_path + +MAX_AMOUNT = 10**15 +MIGRATION = '0037_credits_ledger.sql' +TABLES = ('credits_balances', 'credits_entries', 'credits_audit') +_warned: set[str] = set() +_warn_lock = threading.Lock() +logger = logging.getLogger(__name__) + + +class CreditsError(Exception): + def __init__(self, code='credits_unavailable', status=503): + super().__init__(code) + self.code, self.status = code, status + + +class CreditDenied(AuthorityDenied): + """Bounded error metadata lets request admission return a JSON denial.""" + def __init__(self, code, status=503): + super().__init__(code) + self.code, self.status = code, status + + +def _warn(name): + with _warn_lock: + if name not in _warned: + _warned.add(name) + logger.warning('Invalid %s; credits disabled', name) + + +def enabled(env: Mapping | None = None) -> bool: + values = os.environ if env is None else env + raw = values.get('CREDITS_ENABLED', '') + flag = raw.strip().lower() if isinstance(raw, str) else 'invalid' + if flag in {'', '0', 'false', 'no', 'off'}: + return False + if flag not in {'1', 'true', 'yes', 'on'}: + _warn('CREDITS_ENABLED') + return False + currency = values.get('CREDITS_CURRENCY', '') + if currency == '': + currency = 'USD' + if currency != 'USD': + _warn('CREDITS_CURRENCY') + return False + return True + + +def integer(value, *, nonnegative=False): + if type(value) is not int or not (-MAX_AMOUNT <= value <= MAX_AMOUNT) or (nonnegative and value < 0): + raise CreditsError('invalid_credits_request', 400) + return value + + +def label(value): + if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_:.\-]{1,128}', value): + raise CreditsError('invalid_credits_request', 400) + return value + + +def reason_text(value): + if not isinstance(value, str) or not 1 <= len(value) <= 512 or any(ord(c) < 32 or ord(c) > 126 for c in value): + raise CreditsError('invalid_credits_request', 400) + return value + + +def owner_id(value): + if not isinstance(value, str) or not 1 <= len(value) <= 512 or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise CreditsError('invalid_credits_request', 400) + return value + + +def context_owner(context: TenantContext) -> str: + if type(context) is not TenantContext: + raise CreditsError('invalid_credits_request', 400) + if context.org_id is None: + return owner_id(context.principal_id) + # JSON tuple delimiters cannot collide with legacy opaque principal IDs. + return 'tenant:' + json.dumps([context.org_id, context.team_id, context.principal_id], separators=(',', ':')) + + +def request_fields(owner, kind, amount_microusd, operation_id, revision, *, scoped_id=None, + reference=None, tariff_revision=None, actor=None, reason=None, unknown=False): + owner_id(owner) + label(operation_id) + integer(revision, nonnegative=True) + integer(amount_microusd) + if kind not in {'credit', 'reserve', 'commit', 'release', 'adjust', 'compensate'} or type(unknown) is not bool: + raise CreditsError('invalid_credits_request', 400) + for value in (scoped_id, reference, actor): + if value is not None: + label(value) + if kind in {'reserve', 'commit', 'release'}: + label(scoped_id) + integer(amount_microusd, nonnegative=True) + if kind in {'reserve', 'commit'} and not unknown: + if tariff_revision is None: + raise CreditsError('credits_unpriced', 503) + integer(tariff_revision, nonnegative=True) + elif tariff_revision is not None: + integer(tariff_revision, nonnegative=True) + if kind == 'credit': + integer(amount_microusd, nonnegative=True) + if kind in {'adjust', 'compensate'}: + label(actor) + reason_text(reason) + elif reason is not None: + reason_text(reason) + if kind == 'compensate': + label(reference) + elif reference is not None: + raise CreditsError('invalid_credits_request', 400) + if unknown and (kind != 'reserve' or amount_microusd != 0): + raise CreditsError('invalid_credits_request', 400) + return dict(owner=owner, kind=kind, amount_microusd=amount_microusd, operation_id=operation_id, + revision=revision, scoped_id=scoped_id, reference=reference, tariff_revision=tariff_revision, + actor=actor, reason=reason, unknown=unknown) + + +def deltas(fields, entries): + kind, amount = fields['kind'], fields['amount_microusd'] + balance_delta, held_delta = 0, 0 + if kind in {'credit', 'adjust'}: + balance_delta = amount + elif kind == 'compensate': + original = next((row for row in entries if row['operation_id'] == fields['reference']), None) + if (not original or original['kind'] not in {'credit', 'adjust', 'commit'} + or amount != -original['balance_delta'] + or any(row['kind'] == 'compensate' and row['reference'] == fields['reference'] for row in entries)): + raise CreditsError('credits_conflict', 409) + balance_delta = amount + else: + attempt = [row for row in entries if row['scoped_id'] == fields['scoped_id']] + hold = next((row for row in attempt if row['kind'] == 'reserve' and row['reference'] is None), None) + terminal = any(row['kind'] in {'commit', 'release'} for row in attempt) + if kind == 'reserve' and not fields['unknown']: + if hold: + raise CreditsError('credits_conflict', 409) + held_delta = amount + else: + if not hold or terminal: + raise CreditsError('credits_conflict', 409) + fields['reference'] = hold['operation_id'] + if kind == 'commit': + balance_delta, held_delta = -amount, -hold['amount_microusd'] + elif kind == 'release': + if amount != hold['amount_microusd']: + raise CreditsError('invalid_credits_request', 400) + held_delta = -amount + return balance_delta, held_delta + + +def pagination(cursor, limit): + if (not isinstance(cursor, str) or not re.fullmatch(r'[0-9]{1,16}', cursor) + or int(cursor) > MAX_AMOUNT or type(limit) is not int or not 1 <= limit <= 100): + raise CreditsError('invalid_credits_request', 400) + return int(cursor) + + +def public_entry(row): + return {key: row[key] for key in ('operation_id', 'revision', 'kind', 'amount_microusd', + 'balance_delta', 'held_delta', 'scoped_id', 'reference', 'tariff_revision')} + + +class SqlCreditsLedger: + """SQLite transactions serialize CAS and preserve event/balance atomicity.""" + def __init__(self, path=None, *, initialize=False): + self.path = Path(path) if path is not None else storage_path('USAGE_DB_PATH', 'usage.sqlite3') + if initialize: + self._run(lambda db: db.executescript((Path(__file__).resolve().parents[1] / 'intelligence-migrations' / MIGRATION).read_text()), check=False) + + def _run(self, action, *, check=True): + try: + if check and not self.path.is_file(): + raise CreditsError() + target = self.path.resolve().as_uri() + '?mode=rw' if check else str(self.path) + with closing(sqlite3.connect(target, timeout=10, uri=check)) as db, db: + db.row_factory = sqlite3.Row + db.execute('BEGIN IMMEDIATE') + if check: + db.execute('SELECT owner,operation_id,revision,kind,amount_microusd,balance_delta,held_delta,scoped_id,reference,tariff_revision,actor,reason,document FROM credits_entries LIMIT 0') + db.execute('SELECT owner,operation_id,revision,actor,kind FROM credits_audit LIMIT 0') + db.execute('SELECT balance_microusd, held_microusd, revision FROM credits_balances LIMIT 0') + return action(db) + except (sqlite3.Error, OSError): + raise CreditsError() from None + + def append(self, owner, kind, amount_microusd, operation_id, revision, **kwargs): + fields = request_fields(owner, kind, amount_microusd, operation_id, revision, **kwargs) + document = json.dumps(fields, sort_keys=True, ensure_ascii=False, separators=(',', ':')) + def write(db): + existing = db.execute('SELECT * FROM credits_entries WHERE owner=? AND operation_id=?', (owner, operation_id)).fetchone() + if existing: + if existing['document'] != document: + raise CreditsError('credits_conflict', 409) + return public_entry(existing) + row = db.execute('SELECT * FROM credits_balances WHERE owner=?', (owner,)).fetchone() + balance, held, current = (row['balance_microusd'], row['held_microusd'], row['revision']) if row else (0, 0, 0) + if current != revision: + raise CreditsError('credits_revision_mismatch', 412) + entries = [dict(item) for item in db.execute('SELECT * FROM credits_entries WHERE owner=? AND (scoped_id=? OR operation_id=? OR reference=?)', + (owner, fields['scoped_id'], fields['reference'], fields['reference']))] + bd, hd = deltas(fields, entries) + integer(balance + bd) + integer(held + hd, nonnegative=True) + integer(revision + 1, nonnegative=True) + available = balance + bd - held - hd + reservation = kind == 'reserve' and not fields['unknown'] + enforced_spend = reservation or (kind == 'adjust' and amount_microusd < 0) + if enforced_spend and (available < 0 or (reservation and balance - held <= 0)): + raise CreditsError('credits_insufficient', 409) + db.execute('INSERT OR IGNORE INTO credits_balances(owner) VALUES (?)', (owner,)) + changed = db.execute('UPDATE credits_balances SET balance_microusd=?, held_microusd=?, revision=revision+1 WHERE owner=? AND revision=?', + (balance + bd, held + hd, owner, revision)).rowcount + if changed != 1: + raise CreditsError('credits_revision_mismatch', 412) + db.execute('''INSERT INTO credits_entries + (owner,operation_id,revision,kind,amount_microusd,balance_delta,held_delta,scoped_id,reference,tariff_revision,actor,reason,document) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)''', + (owner, operation_id, revision + 1, kind, amount_microusd, bd, hd, fields['scoped_id'], fields['reference'], + fields['tariff_revision'], fields['actor'], fields['reason'], document)) + if fields['actor']: + db.execute('INSERT INTO credits_audit VALUES (?,?,?,?,?)', (owner, operation_id, revision + 1, fields['actor'], kind)) + return public_entry(db.execute('SELECT * FROM credits_entries WHERE owner=? AND operation_id=?', (owner, operation_id)).fetchone()) + return self._run(write) + + def read(self, owner, *, cursor='0', limit=100): + owner_id(owner) + after = pagination(cursor, limit) + def read(db): + row = db.execute('SELECT * FROM credits_balances WHERE owner=?', (owner,)).fetchone() + balance, held, revision = (row['balance_microusd'], row['held_microusd'], row['revision']) if row else (0, 0, 0) + rows = db.execute('SELECT * FROM credits_entries WHERE owner=? AND revision>? ORDER BY revision LIMIT ?', (owner, after, limit + 1)).fetchall() + page = rows[:limit] + return dict(currency='USD', balance_microusd=balance, held_microusd=held, available_microusd=balance-held, + revision=revision, entries=[public_entry(item) for item in page], + next_cursor=str(page[-1]['revision']) if len(rows) > limit else None) + return self._run(read) + + +class D1CreditsLedger: + """Private transport without retries or local fallback on authority failure.""" + def __init__(self, call): + self.call = call + + def _call(self, operation, **values): + try: + result = self.call({'version': 1, 'operation': operation, **values}) + if not isinstance(result, dict) or type(result.get('version')) is not int or result.get('version') != 1: + raise CreditsError() + if 'error' in result: + code = result['error'].get('code') + statuses = {'credits_conflict': 409, 'credits_insufficient': 409, 'credits_revision_mismatch': 412, + 'invalid_credits_request': 400, 'credits_unpriced': 503} + raise CreditsError(code if code in statuses else 'credits_unavailable', statuses.get(code, 503)) + try: + return _validate_private_result(operation, result, values) + except CreditsError: + raise CreditsError() from None + except CreditsError: + raise + except Exception as error: + statuses = {'credits_conflict': 409, 'credits_insufficient': 409, 'credits_revision_mismatch': 412, + 'invalid_credits_request': 400, 'credits_unpriced': 503} + code = getattr(error, 'code', None) + raise CreditsError(code if code in statuses else 'credits_unavailable', statuses.get(code, 503)) from None + + def append(self, owner, kind, amount_microusd, operation_id, revision, **kwargs): + fields = request_fields(owner, kind, amount_microusd, operation_id, revision, **kwargs) + return self._call('append', **fields) + + def read(self, owner, *, cursor='0', limit=100): + owner_id(owner) + pagination(cursor, limit) + return self._call('read', owner=owner, cursor=cursor, limit=limit) + + +def _validate_entry(value): + fields = {'operation_id', 'revision', 'kind', 'amount_microusd', 'balance_delta', 'held_delta', + 'scoped_id', 'reference', 'tariff_revision'} + if not isinstance(value, dict) or set(value) != fields: + raise CreditsError() + label(value['operation_id']) + integer(value['revision'], nonnegative=True) + for name in ('amount_microusd', 'balance_delta', 'held_delta'): + integer(value[name]) + for name in ('scoped_id', 'reference'): + if value[name] is not None: + label(value[name]) + if value['tariff_revision'] is not None: + integer(value['tariff_revision'], nonnegative=True) + kind, amount, bd, hd = (value[name] for name in ('kind', 'amount_microusd', 'balance_delta', 'held_delta')) + if kind in {'credit', 'adjust', 'compensate'}: + valid = bd == amount and hd == 0 and (kind != 'credit' or amount >= 0) + elif kind == 'reserve': + valid = amount >= 0 and bd == 0 and hd in {0, amount} and value['scoped_id'] is not None + elif kind in {'commit', 'release'}: + valid = amount >= 0 and hd <= 0 and bd == (-amount if kind == 'commit' else 0) and value['reference'] is not None + else: + valid = False + if not valid: + raise CreditsError() + + +def _validate_private_result(operation, result, values): + name = 'entry' if operation == 'append' else 'summary' + if set(result) != {'version', name}: + raise CreditsError() + value = result[name] + if operation == 'append': + _validate_entry(value) + if (any(value[key] != values[key] for key in ('operation_id', 'kind', 'amount_microusd', 'scoped_id', 'tariff_revision')) + or value['revision'] != values['revision'] + 1): + raise CreditsError() + return value + if not isinstance(value, dict) or set(value) != {'currency', 'balance_microusd', 'held_microusd', 'available_microusd', 'revision', 'entries', 'next_cursor'}: + raise CreditsError() + for field in ('balance_microusd', 'held_microusd', 'available_microusd', 'revision'): + integer(value[field], nonnegative=field in {'held_microusd', 'revision'}) + if (value['currency'] != 'USD' or value['available_microusd'] != value['balance_microusd'] - value['held_microusd'] + or not isinstance(value['entries'], list) or len(value['entries']) > values['limit']): + raise CreditsError() + after = int(values['cursor']) + for entry in value['entries']: + _validate_entry(entry) + if not after < entry['revision'] <= value['revision']: + raise CreditsError() + after = entry['revision'] + if value['next_cursor'] is not None and (not value['entries'] or value['next_cursor'] != str(after) or len(value['entries']) != values['limit']): + raise CreditsError() + return value + + +def open_store(): + from services import control_state_d1 + if control_state_d1.using_d1(): + def call(document): + values = {key: value for key, value in document.items() if key not in {'version', 'operation'}} + return control_state_d1.call('credits', document['operation'], **values) + return D1CreditsLedger(call) + if os.environ.get('INTELLIGENCE_STORAGE_BACKEND', '').strip(): + raise CreditsError() + return SqlCreditsLedger() + + +class LedgerCreditAuthority: + """Tariff returns its reviewed revision, or None when cost is unknown/unpriced.""" + def __init__(self, store, tariff): + self.store, self.tariff = store, tariff + + def _apply(self, operation, phase): + if type(operation) is not AuthorityOperation: + raise TypeError('AuthorityOperation required') + if not enabled(): + raise CreditDenied('credits_disabled', 404) + try: + tariff_revision = self.tariff(operation, phase) + unknown = phase == 'reconcile' and tariff_revision is None + if tariff_revision is None and not unknown: + raise CreditDenied('credits_unpriced') + kind = 'reserve' if phase == 'reserve' or unknown else 'commit' + entry = self.store.append(context_owner(operation.context), kind, 0 if unknown else operation.amount, + operation.operation_id, operation.revision, scoped_id=operation.scoped_id, + tariff_revision=tariff_revision, unknown=unknown) + return AuthorityResult(operation.context, operation.scoped_id, entry['revision'], operation.operation_id, True) + except CreditsError as error: + raise CreditDenied(error.code, error.status) from None + + def reserve(self, operation): + return self._apply(operation, 'reserve') + + def commit(self, operation): + return self._apply(operation, 'commit') + + def reconcile(self, operation): + return self._apply(operation, 'reconcile') + + +def register_credit_authority(store, *, tariff): + if not callable(tariff): + raise TypeError('Explicit operator tariff required') + return LedgerCreditAuthority(store, tariff) + + +def register_enterprise_credit(authority): + return register_enterprise_adapters(credit=authority) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..ec62ec44 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -40,6 +40,11 @@ "app.route('/users//rotate-key', methods=['POST'])", "app.route('/v1/metrics/prometheus', methods=['GET', 'OPTIONS'])" ], + "routes/credits.py": [ + "app.get('/admin/credits/')", + "app.post('/admin/credits/adjust')", + "app.route('/v1/credits', methods=['GET', 'OPTIONS'])" + ], "routes/dashboard_security.py": [ "app.route('/admin/audit', endpoint='audit_log')", "app.route(dashboard_sso.ACCESS_LOGIN_PATH, methods=['GET', 'POST'])" @@ -262,6 +267,7 @@ "worker/control-state-d1.mjs", "worker/control-users-d1.mjs", "worker/cors-policy.mjs", + "worker/credits-d1.mjs", "worker/d1-schema.mjs", "worker/enterprise-contract.mjs", "worker/exact-generation-cache.mjs", diff --git a/tests/test_credits_ledger.py b/tests/test_credits_ledger.py new file mode 100644 index 00000000..4d3cb02f --- /dev/null +++ b/tests/test_credits_ledger.py @@ -0,0 +1,398 @@ +"""Local ledger, registered HTTP routes and immutable authority contracts.""" +import json +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest + +from services import credits_ledger as ledger +from services.enterprise_contract import AuthorityOperation, TenantContext, AuthorityDenied, call_authority + + +def vectors(): + text = (Path(__file__).resolve().parents[1] / 'docs/credits-ledger.md').read_text() + return json.loads(text.split('```json\n', 1)[1].split('```', 1)[0]) + + +@pytest.fixture +def store(tmp_path, monkeypatch): + monkeypatch.setenv('CREDITS_ENABLED', 'true') + monkeypatch.setenv('CREDITS_CURRENCY', 'USD') + return ledger.SqlCreditsLedger(tmp_path / 'credits.sqlite3', initialize=True) + + +def append(store, kind, amount, op, revision, **fields): + return store.append('alice', kind, amount, op, revision, **fields) + + +def test_shared_vectors(store): + for vector in vectors(): + if 'invalid_amount' in vector: + with pytest.raises(ledger.CreditsError) as error: + append(store, 'credit', vector['invalid_amount'], 'bad', 0) + assert error.value.status == 400 + continue + values = vector['request'] + if vector.get('error'): + with pytest.raises(ledger.CreditsError) as error: + store.append(**values) + assert error.value.code == vector['error'] + else: + result = store.append(**values) + assert result['revision'] == vector['revision'] + if 'summary' in vector: + summary = store.read(values['owner']) + assert {key: summary[key] for key in vector['summary']} == vector['summary'] + summary = store.read('alice') + assert (summary['balance_microusd'], summary['held_microusd'], summary['available_microusd']) == (75, 0, 75) + + +def test_concurrent_reserves_and_overflow(store): + append(store, 'credit', 100, 'fund', 0) + def reserve(op): + try: + return append(store, 'reserve', 80, op, 1, scoped_id=op, tariff_revision=1) + except ledger.CreditsError as error: + return error.code + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(reserve, ['hedge', 'shadow'])) + assert sum(isinstance(result, dict) for result in results) == 1 + assert store.read('alice')['available_microusd'] == 20 + with pytest.raises(ledger.CreditsError): + append(store, 'credit', ledger.MAX_AMOUNT, 'overflow', 2) + assert store.read('alice')['revision'] == 2 + + +def test_unknown_hold_and_each_attempt_charge(store): + append(store, 'credit', 100, 'fund', 0) + authority = ledger.register_credit_authority(store, tariff=lambda op, phase: None if phase == 'reconcile' else 7) + adapters = ledger.register_enterprise_credit(authority) + def op(rev, name, amount, attempt='hedge'): + return AuthorityOperation(TenantContext('alice'), attempt, rev, name, amount) + call_authority(adapters, 'credit', 'reserve', op(1, 'hold1', 50)) + call_authority(adapters, 'credit', 'reconcile', op(2, 'unknown1', 0)) + assert store.read('alice')['held_microusd'] == 50 + call_authority(adapters, 'credit', 'commit', op(3, 'charge1', 20)) + call_authority(adapters, 'credit', 'reserve', op(4, 'hold2', 40, 'batch')) + call_authority(adapters, 'credit', 'commit', op(5, 'charge2', 30, 'batch')) + assert store.read('alice')['balance_microusd'] == 50 + denied = ledger.register_credit_authority(store, tariff=lambda *_: None) + with pytest.raises(AuthorityDenied, match='credits_unpriced'): + denied.reserve(op(6, 'unpriced', 0, 'new')) + + +def test_compensation_is_linked_and_original_unchanged(store): + original = append(store, 'credit', 100, 'fund', 0) + append(store, 'adjust', -20, 'debit', 1, reason='reviewed', actor='admin') + append(store, 'compensate', 20, 'correction', 2, reference='debit', reason='correction', actor='admin') + assert store.read('alice')['balance_microusd'] == 100 + assert store.read('alice')['entries'][0] == original + with pytest.raises(ledger.CreditsError): + append(store, 'compensate', 20, 'double', 3, reference='debit', reason='correction', actor='admin') + assert store.read('bob')['entries'] == [] + assert store.read('alice', limit=1)['next_cursor'] == '1' + assert store.read('alice', cursor='1', limit=1)['entries'][0]['operation_id'] == 'debit' + + +def test_missing_schema_and_default_off(tmp_path, caplog): + missing = ledger.SqlCreditsLedger(tmp_path / 'missing.sqlite3') + with pytest.raises(ledger.CreditsError, match='credits_unavailable'): + missing.append('alice', 'credit', 1, 'fund', 0) + assert not missing.path.exists() + for flag in ['', 'false', '0', 'bad']: + assert not ledger.enabled({'CREDITS_ENABLED': flag}) + assert ledger.enabled({'CREDITS_ENABLED': 'true', 'CREDITS_CURRENCY': ''}) + for _ in range(2): + assert not ledger.enabled({'CREDITS_ENABLED': 'true', 'CREDITS_CURRENCY': 'EUR'}) + assert caplog.text.count('Invalid CREDITS_CURRENCY') == 1 + + +def test_migration_preserves_old_rows_and_missing_columns_fail_closed(tmp_path): + path = tmp_path / 'old.sqlite3' + migration = (Path(__file__).resolve().parents[1] / 'intelligence-migrations' / ledger.MIGRATION).read_text() + with sqlite3.connect(path) as db: + db.execute('CREATE TABLE old_usage (cost INTEGER)') + db.execute('INSERT INTO old_usage VALUES (NULL)') + db.executescript(migration) + db.executescript(migration) + assert db.execute('SELECT cost FROM old_usage').fetchone() == (None,) + broken = tmp_path / 'broken.sqlite3' + with sqlite3.connect(broken) as db: + db.execute('CREATE TABLE credits_balances (owner TEXT PRIMARY KEY, balance_microusd INTEGER, held_microusd INTEGER, revision INTEGER)') + db.execute('CREATE TABLE credits_entries (owner TEXT)') + db.execute('CREATE TABLE credits_audit (owner TEXT)') + store = ledger.SqlCreditsLedger(broken) + for action in (lambda: store.read('alice'), lambda: append(store, 'credit', 1, 'fund', 0)): + with pytest.raises(ledger.CreditsError, match='credits_unavailable'): + action() + with sqlite3.connect(broken) as db: + assert db.execute('SELECT count(*) FROM credits_balances').fetchone()[0] == 0 + + +def test_spend_above_estimate_closes_only_its_hold(store): + append(store, 'credit', 100, 'fund', 0) + append(store, 'reserve', 30, 'hedge', 1, scoped_id='hedge', tariff_revision=1) + append(store, 'reserve', 60, 'shadow', 2, scoped_id='shadow', tariff_revision=1) + append(store, 'commit', 41, 'charge', 3, scoped_id='hedge', tariff_revision=1) + summary = store.read('alice') + assert (summary['balance_microusd'], summary['held_microusd'], summary['available_microusd']) == (59, 60, -1) + append(store, 'reserve', 0, 'unknown', 4, scoped_id='shadow', unknown=True) + assert store.read('alice')['held_microusd'] == 60 + append(store, 'release', 60, 'release', 5, scoped_id='shadow') + with pytest.raises(ledger.CreditsError, match='credits_conflict'): + append(store, 'commit', 1, 'late', 6, scoped_id='shadow', tariff_revision=1) + assert store.read('alice')['balance_microusd'] == 59 + + +@pytest.mark.parametrize('recovery', ['credit', 'adjust']) +def test_commit_overrun_blocks_new_spend_until_funded(store, recovery): + append(store, 'credit', 100, 'fund', 0) + authority = ledger.register_credit_authority(store, tariff=lambda *_: 1) + def op(revision, name, amount, attempt='attempt'): + return AuthorityOperation(TenantContext('alice'), attempt, revision, name, amount) + authority.reserve(op(1, 'hold', 60)) + with pytest.raises(ledger.CreditDenied, match='credits_revision_mismatch') as error: + authority.commit(op(1, 'charge', 150)) + assert error.value.status == 412 + charged = authority.commit(op(2, 'charge', 150)) + assert authority.commit(op(2, 'charge', 150)) == charged + summary = store.read('alice') + assert (summary['balance_microusd'], summary['held_microusd'], summary['available_microusd']) == (-50, 0, -50) + assert summary['revision'] == 3 + assert ledger.D1CreditsLedger(lambda _: dict(version=1, summary=summary)).read('alice') == summary + for amount in (0, 1): + with pytest.raises(ledger.CreditDenied, match='credits_insufficient'): + authority.reserve(op(3, f'denied{amount}', amount, f'new{amount}')) + with pytest.raises(ledger.CreditsError, match='credits_insufficient'): + append(store, 'adjust', -1, 'debit', 3, actor='admin', reason='reviewed') + with pytest.raises(ledger.CreditDenied, match='credits_conflict'): + authority.commit(op(3, 'late', 1)) + with pytest.raises(ledger.CreditDenied, match='credits_conflict'): + authority.commit(op(2, 'charge', 151)) + extra = dict(actor='admin', reason='reviewed') if recovery == 'adjust' else {} + append(store, recovery, 25, 'partial', 3, **extra) + assert store.read('alice')['available_microusd'] == -25 + append(store, recovery, 25, 'zero', 4, **extra) + with pytest.raises(ledger.CreditDenied, match='credits_insufficient'): + authority.reserve(op(5, 'zero-denied', 0, 'zero-attempt')) + append(store, recovery, 50, 'recovered', 5, **extra) + assert store.read('alice')['available_microusd'] == 50 + authority.reserve(op(6, 'next', 1, 'next')) + assert store.read('alice')['available_microusd'] == 49 + + +@pytest.mark.parametrize('balance,held', [(-ledger.MAX_AMOUNT, 0), (-50, 10), (ledger.MAX_AMOUNT, 0)]) +def test_private_summary_accepts_signed_balances(store, balance, held): + summary = dict(currency='USD', balance_microusd=balance, held_microusd=held, + available_microusd=balance-held, revision=0, entries=[], next_cursor=None) + assert ledger.D1CreditsLedger(lambda _: dict(version=1, summary=summary)).read('alice') == summary + invalid = [dict(held_microusd=-1), dict(revision=-1), dict(balance_microusd=-ledger.MAX_AMOUNT-1), + dict(available_microusd=-ledger.MAX_AMOUNT-1), dict(balance_microusd=True)] + for change in invalid: + with pytest.raises(ledger.CreditsError, match='credits_unavailable'): + ledger.D1CreditsLedger(lambda _: dict(version=1, summary={**summary, **change})).read('alice') + + +def test_duplicate_concurrency_pagination_and_content_free_audit(store): + def fund(_): + return append(store, 'adjust', 100, 'fund', 0, actor='admin', reason='Reviewed allocation') + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(fund, range(2))) + assert results[0] == results[1] + for revision in range(1, 102): + append(store, 'adjust', 1, f'grant{revision}', revision, actor='admin', reason='Reviewed allocation') + page = store.read('alice') + assert len(page['entries']) == 100 + assert page['next_cursor'] == '100' + assert len(store.read('alice', cursor='100')['entries']) == 2 + with sqlite3.connect(store.path) as db: + assert [field[1] for field in db.execute('PRAGMA table_info(credits_audit)')] == ['owner', 'operation_id', 'revision', 'actor', 'kind'] + assert db.execute('SELECT count(*) FROM credits_audit').fetchone()[0] == 102 + balance, held = db.execute('SELECT SUM(balance_delta),SUM(held_delta) FROM credits_entries').fetchone() + assert (balance, held) == (page['balance_microusd'], page['held_microusd']) + for kwargs in ({'limit': 101}, {'limit': True}, {'cursor': '-1'}, {'cursor': '1.5'}): + with pytest.raises(ledger.CreditsError): + store.read('alice', **kwargs) + + +def test_tenant_namespace_and_d1_outage_have_no_local_fallback(store): + context = TenantContext('alice', org_id='org', team_id='team') + owner = ledger.context_owner(context) + store.append(owner, 'credit', 100, 'fund', 0) + assert store.read('alice')['balance_microusd'] == 0 + assert owner != ledger.context_owner(TenantContext('alice', org_id='other', team_id='team')) + d1 = ledger.D1CreditsLedger(lambda _: (_ for _ in ()).throw(OSError('private transport unavailable'))) + with pytest.raises(ledger.CreditsError, match='credits_unavailable'): + d1.read('alice') + assert store.read('alice')['balance_microusd'] == 0 + + +def test_private_transport_validates_response_and_preserves_errors(store): + entry = append(store, 'credit', 100, 'fund', 0) + response = dict(version=1, entry=entry) + d1 = ledger.D1CreditsLedger(lambda _: response) + assert d1.append('alice', 'credit', 100, 'fund', 0) == entry + for altered in ({**entry, 'amount_microusd': True}, {**entry, 'operation_id': 'foreign'}, + {**entry, 'revision': 2}, {**entry, 'prompt': 'private'}, {**entry, 'balance_delta': 99}): + with pytest.raises(ledger.CreditsError) as error: + ledger.D1CreditsLedger(lambda _: dict(version=1, entry=altered)).append('alice', 'credit', 100, 'fund', 0) + assert error.value.code == 'credits_unavailable' and error.value.status == 503 + summary = store.read('alice') + assert ledger.D1CreditsLedger(lambda _: dict(version=1, summary=summary)).read('alice') == summary + for altered in ({**summary, 'available_microusd': 101}, {**summary, 'next_cursor': 'bogus'}, {**summary, 'prompt': 'private'}): + with pytest.raises(ledger.CreditsError, match='credits_unavailable'): + ledger.D1CreditsLedger(lambda _: dict(version=1, summary=altered)).read('alice') + with pytest.raises(ledger.CreditsError) as error: + ledger.D1CreditsLedger(lambda _: dict(version=1, error={'code': 'credits_revision_mismatch'})).read('alice') + assert error.value.status == 412 + + +def test_disabled_authority_never_prices_or_changes_storage(store, monkeypatch): + def forbidden(*args): + pytest.fail('disabled tariff evaluated') + authority = ledger.register_credit_authority(store, tariff=forbidden) + monkeypatch.setenv('CREDITS_ENABLED', '') + with pytest.raises(AuthorityDenied, match='credits_disabled'): + authority.reserve(AuthorityOperation(TenantContext('alice'), 'attempt', 0, 'disabled', 0)) + assert store.read('alice')['revision'] == 0 + + +def test_registered_authority_denial_stops_provider_before_handoff(tmp_path, monkeypatch): + from flask import Flask + from types import SimpleNamespace + from routes import credits + monkeypatch.setenv('CREDITS_ENABLED', 'true') + monkeypatch.setenv('CREDITS_CURRENCY', 'USD') + store = ledger.SqlCreditsLedger(tmp_path / 'missing.sqlite3') + authority = ledger.register_credit_authority(store, tariff=lambda *_: 1) + app = Flask('credits-admission') + credits.register_credits_routes(app, SimpleNamespace(exempt=lambda fn: fn), store=store) + provider_calls = [] + def generation(): + authority.reserve(AuthorityOperation(TenantContext('alice'), 'attempt', 0, 'hold', 1)) + provider_calls.append('called') + return 'provider result' + app.add_url_rule('/generation', 'generation', generation) + response = app.test_client().get('/generation') + assert response.status_code == 503 and response.json['error']['code'] == 'credits_unavailable' + assert provider_calls == [] and not store.path.exists() + + +def test_registered_real_auth_scope_and_missing_schema(store, tmp_path, monkeypatch): + from flask import Flask + from types import SimpleNamespace + from routes import credits + helpers = credits.api_authenticate_only.__globals__ + def authenticate(key, address): + if key == 'models-test': + return {'username': 'alice', 'scopes': ['models']} + if key == 'chat-test': + return {'username': 'alice', 'scopes': ['chat']} + return None + monkeypatch.setattr(helpers['AuthService'], 'verify_api_key', authenticate) + monkeypatch.setattr(helpers['request_accounting'], 'check_key_controls', lambda user: None) + monkeypatch.setattr(helpers['request_accounting'], 'begin', lambda: None) + app = Flask(__name__) + app.config['SECRET_KEY'] = 'synthetic-test-secret' + credits.register_credits_routes(app, SimpleNamespace(exempt=lambda fn: fn), store=store) + client = app.test_client() + append(store, 'credit', 100, 'fund', 0) + assert client.get('/v1/credits', headers={'Authorization': 'Bearer chat-test'}).status_code == 403 + assert client.get('/v1/credits').status_code == 401 + assert client.get('/v1/credits?owner=bob', headers={'Authorization': 'Bearer models-test'}).json['balance_microusd'] == 100 + assert client.get('/v1/credits?limit=101', headers={'Authorization': 'Bearer models-test'}).status_code == 400 + assert client.get('/v1/credits?cursor=-1', headers={'Authorization': 'Bearer models-test'}).status_code == 400 + missing = ledger.SqlCreditsLedger(tmp_path / 'absent.sqlite3') + other = Flask('missing-credits') + other.config['SECRET_KEY'] = 'synthetic-test-secret' + credits.register_credits_routes(other, SimpleNamespace(exempt=lambda fn: fn), store=missing) + response = other.test_client().get('/v1/credits', headers={'Authorization': 'Bearer models-test'}) + assert response.status_code == 503 and response.json['error']['code'] == 'credits_unavailable' + assert not missing.path.exists() + + +def test_real_admin_session_csrf_audit_and_disabled_response_bytes(store, monkeypatch): + from flask import Flask + from flask_wtf.csrf import CSRFProtect, generate_csrf + from routes import credits + session_authority = credits.login_required.__globals__['AuthService'] + admin_authority = credits.require_admin_dashboard_user.__globals__['AuthService'] + signed_in = {'value': False} + user = {'username': 'member', 'is_admin': False} + monkeypatch.setattr(session_authority, 'is_authenticated', lambda: signed_in['value']) + monkeypatch.setattr(admin_authority, 'get_current_user', lambda: user) + app = Flask('credits-admin-security') + app.config.update(SECRET_KEY='synthetic-test-secret', TESTING=True) + app.add_url_rule('/login', 'login', lambda: 'login') + app.add_url_rule('/csrf-token', 'token', lambda: {'csrf_token': generate_csrf()}) + app.add_url_rule('/existing', 'existing', lambda: ('unchanged', 201, {'X-Existing': 'value'})) + csrf = CSRFProtect(app) + credits.register_credits_routes(app, csrf, store=store) + client = app.test_client() + assert client.get('/admin/credits/alice').status_code == 302 + signed_in['value'] = True + assert client.get('/admin/credits/alice').status_code == 403 + token = client.get('/csrf-token').json['csrf_token'] + body = dict(owner='alice', amount_microusd=100, operation_id='fund', reason='Reviewed allocation') + assert client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"0"', 'X-CSRFToken': token}).status_code == 403 + user.update(username='admin', is_admin=True) + assert client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"0"'}).status_code == 400 + result = client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"0"', 'X-CSRFToken': token}) + assert result.status_code == 200 and result.headers['ETag'] == '"1"' + assert client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"0"', 'X-CSRFToken': token}).json == result.json + with sqlite3.connect(store.path) as db: + assert db.execute('SELECT actor FROM credits_audit').fetchone() == ('admin',) + before = client.get('/existing') + monkeypatch.setenv('CREDITS_ENABLED', '') + after = client.get('/existing') + assert (after.data, after.status_code, dict(after.headers)) == (before.data, before.status_code, dict(before.headers)) + assert client.post('/admin/credits/adjust', json=body).status_code == 404 + assert store.read('alice')['balance_microusd'] == 100 + + +def test_registered_routes_auth_cas_and_no_store(store, monkeypatch): + from flask import Flask, g + from flask_wtf.csrf import CSRFProtect + from routes import credits + import route_helpers + # Decorator collaborators are injected before route registration. + def auth(required_scope): + assert required_scope == 'models' + def decorate(fn): + def wrapped(*args, **kwargs): + from flask import request + if request.headers.get('Authorization') != 'Bearer synthetic': + return {'error': 'unauthorized'}, 401 + g.authenticated_user = {'username': 'alice'} + return fn(*args, **kwargs) + wrapped.__name__ = fn.__name__ + return wrapped + return decorate + monkeypatch.setattr(credits, 'api_authenticate_only', auth) + monkeypatch.setattr(credits, 'login_required', lambda fn: fn) + monkeypatch.setattr(credits, 'require_admin_dashboard_user', lambda: {'username': 'admin'}) + monkeypatch.setattr(route_helpers, 'login_required', lambda fn: fn) + monkeypatch.setenv('CREDITS_ENABLED', 'false') + monkeypatch.setenv('CREDITS_CURRENCY', 'USD') + app = Flask(__name__) + app.config.update(SECRET_KEY='synthetic-test-secret', TESTING=True) + csrf = CSRFProtect(app) + credits.register_credits_routes(app, csrf, store=store) + client = app.test_client() + assert client.get('/v1/credits').status_code == 404 + assert client.post('/admin/credits/adjust', json={}).status_code == 404 + monkeypatch.setenv('CREDITS_ENABLED', 'true') + assert client.get('/v1/credits').status_code == 401 + response = client.get('/v1/credits?owner=bob', headers={'Authorization': 'Bearer synthetic'}) + assert response.json['entries'] == [] + assert response.headers['Cache-Control'] == 'no-store' + assert client.post('/admin/credits/adjust', json={}).status_code == 400 + app.config['WTF_CSRF_ENABLED'] = False + body = dict(owner='alice', amount_microusd=100, operation_id='grant', reason='reviewed') + assert client.post('/admin/credits/adjust', json=body).status_code == 428 + assert client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"1"'}).status_code == 412 + assert client.post('/admin/credits/adjust', json=body, headers={'If-Match': '"0"'}).status_code == 200 + assert client.get('/admin/credits/alice').headers['ETag'] == '"1"' + monkeypatch.setattr(credits, 'require_admin_dashboard_user', lambda: (_ for _ in ()).throw(ledger.CreditsError('admin_required', 403))) + assert client.get('/admin/credits/alice').status_code == 403 diff --git a/tests/test_credits_ledger_worker.mjs b/tests/test_credits_ledger_worker.mjs new file mode 100644 index 00000000..e99b2f01 --- /dev/null +++ b/tests/test_credits_ledger_worker.mjs @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { readFileSync } from 'node:fs'; +import { DatabaseSync } from 'node:sqlite'; +import { spawnSync } from 'node:child_process'; +import { handleCreditsRequest, appendCredit, readCredits, registerCreditAuthority, creditsEnabled, contextOwner } from '../worker/credits-d1.mjs'; +import { AuthorityOperation, TenantContext, callAuthority, registerEnterpriseAdapters } from '../worker/enterprise-contract.mjs'; + +const migration = readFileSync(new URL('../intelligence-migrations/0037_credits_ledger.sql', import.meta.url), 'utf8'); +const vectors = JSON.parse(readFileSync(new URL('../docs/credits-ledger.md', import.meta.url), 'utf8').split('```json\n')[1].split('```')[0]); + +function database(t, schema = true) { + const sql = new DatabaseSync(':memory:'); + t.after(() => sql.close()); + if (schema) sql.exec(migration); + const db = { prepare(query) { + let values = []; + const statement = { + bind(...args) { values = args; return statement; }, + first() { return sql.prepare(query).get(...values) ?? null; }, + all() { + const prepared = sql.prepare(query); + if (prepared.columns().length) return { results: prepared.all(...values), success: true }; + return { results: [], success: true, meta: { changes: Number(prepared.run(...values).changes) } }; + }, + }; + return statement; + }, async batch(statements) { + sql.exec('BEGIN IMMEDIATE'); + try { + const rows = statements.map(statement => statement.all()); + sql.exec('COMMIT'); + return rows; + } catch (error) { sql.exec('ROLLBACK'); throw error; } + } }; + const env = { CREDITS_ENABLED: 'true', CREDITS_CURRENCY: 'USD', INTELLIGENCE_DB: db }; + const call = async body => { + const response = await handleCreditsRequest(new Request('http://intelligence.internal/v1/credits', { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ version: 1, ...body }), + }), env); + return { status: response.status, ...await response.json() }; + }; + return { sql, db, env, call }; +} + +const entry = (kind, amount, operation_id, revision, extra = {}) => ({ owner: 'alice', kind, + amount_microusd: amount, operation_id, revision, ...extra }); + +test('shared integer, insufficient and duplicate operation vectors', async t => { + const { call } = database(t); + for (const vector of vectors) { + const request = vector.request ?? entry('credit', vector.invalid_amount, 'bad', 0); + const result = await call({ operation: 'append', ...request }); + if ('invalid_amount' in vector) assert.equal(result.status, 400); + else if (vector.error) assert.equal(result.error.code, vector.error); + else assert.equal(result.entry.revision, vector.revision); + if (vector.summary) { + const { summary } = await call({ operation: 'read', owner: request.owner }); + for (const [key, value] of Object.entries(vector.summary)) assert.equal(summary[key], value); + } + } + const summary = (await call({ operation: 'read', owner: 'alice' })).summary; + assert.equal(summary.balance_microusd, 75); + assert.equal(summary.held_microusd, 0); + assert.equal(summary.available_microusd, 75); +}); + +test('concurrent CAS reserves and duplicate requests spend once', async t => { + const { db } = database(t); + await appendCredit(db, entry('credit', 100, 'fund', 0)); + const reserve = name => appendCredit(db, entry('reserve', 80, name, 1, { scoped_id: name, tariff_revision: 1 })); + const results = await Promise.allSettled([reserve('hedge'), reserve('shadow')]); + assert.equal(results.filter(result => result.status === 'fulfilled').length, 1); + assert.equal((await readCredits(db, 'alice')).available_microusd, 20); + const { db: other } = database(t); + const fund = entry('credit', 100, 'fund', 0); + const duplicates = await Promise.all([appendCredit(other, fund), appendCredit(other, fund)]); + assert.deepEqual(duplicates[0], duplicates[1]); + assert.equal((await readCredits(other, 'alice')).balance_microusd, 100); +}); + +test('unknown reconciliation retains hold and all attempts settle independently', async t => { + const { db, env } = database(t); + await appendCredit(db, entry('credit', 100, 'fund', 0)); + const authority = registerCreditAuthority(db, { env, tariff: (_op, phase) => phase === 'reconcile' ? null : 7 }); + const adapters = registerEnterpriseAdapters({ credit: authority }); + const op = (revision, operation_id, amount, scoped_id = 'hedge') => new AuthorityOperation({ + context: new TenantContext({ principal_id: 'alice' }), revision, operation_id, amount, scoped_id }); + await callAuthority(adapters, 'credit', 'reserve', op(1, 'hold1', 50)); + await callAuthority(adapters, 'credit', 'reconcile', op(2, 'unknown1', 0)); + assert.equal((await readCredits(db, 'alice')).held_microusd, 50); + await callAuthority(adapters, 'credit', 'commit', op(3, 'charge1', 20)); + await callAuthority(adapters, 'credit', 'reserve', op(4, 'hold2', 40, 'batch')); + await callAuthority(adapters, 'credit', 'commit', op(5, 'charge2', 30, 'batch')); + assert.equal((await readCredits(db, 'alice')).balance_microusd, 50); + await assert.rejects(registerCreditAuthority(db, { env, tariff: () => null }).reserve(op(6, 'unpriced', 0, 'new')), /credits_unpriced/); + const disabled = registerCreditAuthority({ prepare() { assert.fail('disabled storage'); } }, { tariff: () => assert.fail('disabled tariff') }); + await assert.rejects(disabled.reserve(op(6, 'disabled', 0, 'new')), /credits_disabled/); +}); + +test('compensation, releases, overflow and cursor owner isolation', async t => { + const { db } = database(t); + const original = await appendCredit(db, entry('credit', 100, 'fund', 0)); + await appendCredit(db, entry('adjust', -20, 'debit', 1, { actor: 'admin', reason: 'reviewed' })); + await appendCredit(db, entry('compensate', 20, 'correction', 2, { actor: 'admin', reason: 'reviewed', reference: 'debit' })); + await assert.rejects(appendCredit(db, entry('compensate', 20, 'twice', 3, { actor: 'admin', reason: 'reviewed', reference: 'debit' })), /credits_conflict/); + await assert.rejects(appendCredit(db, entry('credit', 1000000000000000, 'overflow', 3)), /invalid_credits_request/); + assert.deepEqual((await readCredits(db, 'alice')).entries[0], original); + assert.deepEqual((await readCredits(db, 'bob')).entries, []); + assert.equal((await readCredits(db, 'alice', { limit: 1 })).next_cursor, '1'); + assert.equal((await readCredits(db, 'alice', { limit: 1, cursor: '1' })).entries[0].operation_id, 'debit'); + await appendCredit(db, entry('reserve', 40, 'hold', 3, { scoped_id: 'new', tariff_revision: 1 })); + await appendCredit(db, entry('release', 40, 'release', 4, { scoped_id: 'new' })); + assert.equal((await readCredits(db, 'alice')).held_microusd, 0); +}); + +test('private boundary, disabled traffic and missing schema fail closed', async t => { + const { call, env, db } = database(t, false); + assert.equal((await call({ operation: 'append', ...entry('credit', 1, 'fund', 0) })).error.code, 'credits_unavailable'); + assert.equal((await call({ operation: 'read', owner: 'alice' })).status, 503); + for (const flag of [undefined, '', 'false', 'bad']) { + const response = await handleCreditsRequest(new Request('http://intelligence.internal/v1/credits', { method: 'POST' }), { + CREDITS_ENABLED: flag, INTELLIGENCE_DB: { prepare() { assert.fail('disabled storage'); } }, + }); + assert.equal(response.status, 404); + } + const foreign = await handleCreditsRequest(new Request('https://example.test/v1/credits', { method: 'POST' }), env); + assert.equal(foreign.status, 404); + const { call: healthy } = database(t); + assert.equal((await healthy({ operation: 'append', ...entry('credit', 1, 'fund', 0), prompt: 'content' })).status, 400); + assert.equal((await healthy({ operation: 'append', ...entry('reserve', 1, 'hold', 0, { scoped_id: 'attempt' }) })).error.code, 'credits_unpriced'); + for (const currency of ['EUR', false, null]) assert.equal(creditsEnabled({ CREDITS_ENABLED: 'true', CREDITS_CURRENCY: currency }), false); + assert.equal(creditsEnabled({ CREDITS_ENABLED: 'true', CREDITS_CURRENCY: '' }), true); + let dispatched = false; + const operation = new AuthorityOperation({ context: new TenantContext({ principal_id: 'alice' }), + scoped_id: 'attempt', operation_id: 'hold', revision: 0, amount: 1 }); + try { + await registerCreditAuthority(db, { env, tariff: () => 1 }).reserve(operation); + dispatched = true; + } catch (error) { + assert.equal(error.status, 503); + assert.equal((await error.response().json()).error.code, 'credits_unavailable'); + } + assert.equal(dispatched, false); +}); + +test('actual spend above estimate closes only its hold', async t => { + const { db } = database(t); + await appendCredit(db, entry('credit', 100, 'fund', 0)); + await appendCredit(db, entry('reserve', 30, 'hedge', 1, { scoped_id: 'hedge', tariff_revision: 1 })); + await appendCredit(db, entry('reserve', 60, 'shadow', 2, { scoped_id: 'shadow', tariff_revision: 1 })); + await appendCredit(db, entry('commit', 41, 'charge', 3, { scoped_id: 'hedge', tariff_revision: 1 })); + const summary = await readCredits(db, 'alice'); + assert.deepEqual([summary.balance_microusd, summary.held_microusd, summary.available_microusd], [59, 60, -1]); + await appendCredit(db, entry('reserve', 0, 'unknown', 4, { scoped_id: 'shadow', unknown: true })); + assert.equal((await readCredits(db, 'alice')).held_microusd, 60); + await appendCredit(db, entry('release', 60, 'release', 5, { scoped_id: 'shadow' })); + await assert.rejects(appendCredit(db, entry('commit', 1, 'late', 6, { scoped_id: 'shadow', tariff_revision: 1 })), /credits_conflict/); + assert.equal((await readCredits(db, 'alice')).balance_microusd, 59); +}); + +for (const recovery of ['credit', 'adjust']) test(`commit overrun blocks new spend until ${recovery} funding`, async t => { + const { db, env, call } = database(t); + await appendCredit(db, entry('credit', 100, 'fund', 0)); + const authority = registerCreditAuthority(db, { env, tariff: () => 1 }); + const op = (revision, operation_id, amount, scoped_id = 'attempt') => new AuthorityOperation({ + context: new TenantContext({ principal_id: 'alice' }), revision, operation_id, amount, scoped_id }); + await authority.reserve(op(1, 'hold', 60)); + await assert.rejects(authority.commit(op(1, 'charge', 150)), error => error.code === 'credits_revision_mismatch' && error.status === 412); + const charged = await authority.commit(op(2, 'charge', 150)); + assert.deepEqual(await authority.commit(op(2, 'charge', 150)), charged); + const { status, summary } = await call({ operation: 'read', owner: 'alice' }); + assert.equal(status, 200); + assert.deepEqual([summary.balance_microusd, summary.held_microusd, summary.available_microusd], [-50, 0, -50]); + assert.equal(summary.revision, 3); + for (const amount of [0, 1]) await assert.rejects(authority.reserve(op(3, `denied${amount}`, amount, `new${amount}`)), /credits_insufficient/); + await assert.rejects(appendCredit(db, entry('adjust', -1, 'debit', 3, { actor: 'admin', reason: 'reviewed' })), /credits_insufficient/); + await assert.rejects(authority.commit(op(3, 'late', 1)), /credits_conflict/); + await assert.rejects(authority.commit(op(2, 'charge', 151)), /credits_conflict/); + const extra = recovery === 'adjust' ? { actor: 'admin', reason: 'reviewed' } : {}; + await appendCredit(db, entry(recovery, 25, 'partial', 3, extra)); + assert.equal((await readCredits(db, 'alice')).available_microusd, -25); + await appendCredit(db, entry(recovery, 25, 'zero', 4, extra)); + await assert.rejects(authority.reserve(op(5, 'zero-denied', 0, 'zero-attempt')), /credits_insufficient/); + await appendCredit(db, entry(recovery, 50, 'recovered', 5, extra)); + assert.equal((await readCredits(db, 'alice')).available_microusd, 50); + await authority.reserve(op(6, 'next', 1, 'next')); + assert.equal((await readCredits(db, 'alice')).available_microusd, 49); +}); + +test('migration rehearsal, incomplete columns, audit and bounded pagination', async t => { + const { db, sql } = database(t); + sql.exec('CREATE TABLE old_usage (cost INTEGER); INSERT INTO old_usage VALUES (NULL)'); + sql.exec(migration); + assert.equal(sql.prepare('SELECT cost FROM old_usage').get().cost, null); + for (let revision = 0; revision < 102; revision++) { + await appendCredit(db, entry('adjust', 1, `grant${revision}`, revision, { actor: 'admin', reason: 'Reviewed allocation' })); + } + const first = await readCredits(db, 'alice'); + assert.equal(first.entries.length, 100); + assert.equal(first.next_cursor, '100'); + assert.equal((await readCredits(db, 'alice', { cursor: '100' })).entries.length, 2); + assert.equal(sql.prepare('SELECT count(*) AS n FROM credits_audit').get().n, 102); + assert.deepEqual(sql.prepare('PRAGMA table_info(credits_audit)').all().map(row => row.name), ['owner', 'operation_id', 'revision', 'actor', 'kind']); + for (const options of [{ limit: 101 }, { limit: true }, { cursor: '-1' }, { cursor: '1.5' }]) await assert.rejects(readCredits(db, 'alice', options), /invalid_credits_request/); + const { db: broken, sql: partial } = database(t, false); + partial.exec(`CREATE TABLE credits_balances (owner TEXT PRIMARY KEY,balance_microusd INTEGER,held_microusd INTEGER,revision INTEGER); + CREATE TABLE credits_entries (owner TEXT); CREATE TABLE credits_audit (owner TEXT)`); + await assert.rejects(readCredits(broken, 'alice'), /credits_unavailable/); + await assert.rejects(appendCredit(broken, entry('credit', 1, 'fund', 0)), /credits_unavailable/); + assert.equal(partial.prepare('SELECT count(*) AS n FROM credits_balances').get().n, 0); + const org = new TenantContext({ principal_id: 'alice', org_id: 'org', team_id: 'team' }); + assert.equal(contextOwner(org), 'tenant:["org","team","alice"]'); + assert.notEqual(contextOwner(org), contextOwner(new TenantContext({ principal_id: 'alice', org_id: 'other', team_id: 'team' }))); +}); + +test('Python and D1 return identical ledger vectors', async t => { + const { db } = database(t); + for (const vector of vectors.filter(vector => vector.request && !vector.error)) await appendCredit(db, vector.request); + const code = `import os,sys,json,tempfile\nfrom pathlib import Path\nsys.path.insert(0,os.getcwd())\nfrom services.credits_ledger import SqlCreditsLedger\ntext=Path('docs/credits-ledger.md').read_text()\nvectors=json.loads(text.split('\x60\x60\x60json\\n',1)[1].split('\x60\x60\x60',1)[0])\nwith tempfile.TemporaryDirectory() as d:\n store=SqlCreditsLedger(Path(d)/'credits.sqlite3',initialize=True)\n for v in vectors:\n if 'request' in v and not v.get('error'): store.append(**v['request'])\n print(json.dumps(store.read('alice')))\n`; + const python = spawnSync('/home/mysterious/storage/github/MultiLLM-Proxy/.venv/bin/python', ['-I', '-c', code], { + cwd: new URL('..', import.meta.url), encoding: 'utf8', timeout: 10000, + }); + assert.equal(python.status, 0, python.stderr); + assert.deepEqual(await readCredits(db, 'alice'), JSON.parse(python.stdout)); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..61c371b6 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,8 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "CREDITS_ENABLED", + "CREDITS_CURRENCY", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/credits-d1.mjs b/worker/credits-d1.mjs new file mode 100644 index 00000000..7e1f0588 --- /dev/null +++ b/worker/credits-d1.mjs @@ -0,0 +1,273 @@ +/** Private append-only integer credits with atomic D1 balance revisions. */ +import { AuthorityDenied, AuthorityOperation, AuthorityResult, TenantContext } from './enterprise-contract.mjs'; + +export const MAX_AMOUNT = 1_000_000_000_000_000; +const warned = new Set(); +const entryKeys = ['operation_id', 'revision', 'kind', 'amount_microusd', 'balance_delta', + 'held_delta', 'scoped_id', 'reference', 'tariff_revision']; + +export class CreditsError extends Error { + constructor(code = 'credits_unavailable', status = 503) { + super(code); this.code = code; this.status = status; + } +} + +export class CreditDenied extends AuthorityDenied { + constructor(code, status = 503) { super(code); this.code = code; this.status = status; } + response() { + return Response.json({ error: { code: this.code, message: 'Credits operation unavailable.' } }, { + status: this.status, headers: { 'Cache-Control': 'no-store' }, + }); + } +} + +export function creditsEnabled(env = {}, warn = message => console.warn(message)) { + const raw = env.CREDITS_ENABLED; + const flag = raw === undefined ? '' : typeof raw === 'string' ? raw.trim().toLowerCase() : 'invalid'; + const invalid = name => { + if (!warned.has(name)) { warned.add(name); warn(`Invalid ${name}; credits disabled`); } + return false; + }; + if (['', '0', 'false', 'no', 'off'].includes(flag)) return false; + if (!['1', 'true', 'yes', 'on'].includes(flag)) return invalid('CREDITS_ENABLED'); + if ((env.CREDITS_CURRENCY === undefined || env.CREDITS_CURRENCY === '' ? 'USD' : env.CREDITS_CURRENCY) !== 'USD') { + return invalid('CREDITS_CURRENCY'); + } + return true; +} + +function integer(value, nonnegative = false) { + if (!Number.isSafeInteger(value) || Math.abs(value) > MAX_AMOUNT || nonnegative && value < 0) { + throw new CreditsError('invalid_credits_request', 400); + } + return value; +} +function label(value) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_:.\-]{1,128}$/.test(value) || /\n|\r/.test(value)) { + throw new CreditsError('invalid_credits_request', 400); + } + return value; +} +function ownerId(value) { + if (typeof value !== 'string' || value.length < 1 || value.length > 512 || /[\x00-\x1f\x7f]/.test(value)) { + throw new CreditsError('invalid_credits_request', 400); + } + return value; +} +function reasonText(value) { + if (typeof value !== 'string' || value.length < 1 || value.length > 512 || /[^\x20-\x7e]/.test(value)) { + throw new CreditsError('invalid_credits_request', 400); + } + return value; +} +export function contextOwner(context) { + if (!(context instanceof TenantContext)) throw new CreditsError('invalid_credits_request', 400); + return context.org_id === null ? ownerId(context.principal_id) + : `tenant:${JSON.stringify([context.org_id, context.team_id, context.principal_id])}`; +} +function requestFields(data) { + const { owner, kind, amount_microusd, operation_id, revision, scoped_id = null, reference = null, + tariff_revision = null, actor = null, reason = null, unknown = false } = data; + ownerId(owner); label(operation_id); integer(amount_microusd); integer(revision, true); + if (!['credit', 'reserve', 'commit', 'release', 'adjust', 'compensate'].includes(kind) || typeof unknown !== 'boolean') { + throw new CreditsError('invalid_credits_request', 400); + } + for (const value of [scoped_id, reference, actor]) if (value !== null) label(value); + if (['reserve', 'commit', 'release'].includes(kind)) { label(scoped_id); integer(amount_microusd, true); } + if (['reserve', 'commit'].includes(kind) && !unknown) { + if (tariff_revision === null) throw new CreditsError('credits_unpriced'); + integer(tariff_revision, true); + } else if (tariff_revision !== null) integer(tariff_revision, true); + if (kind === 'credit') integer(amount_microusd, true); + if (['adjust', 'compensate'].includes(kind)) { label(actor); reasonText(reason); } + else if (reason !== null) reasonText(reason); + if (kind === 'compensate') label(reference); + else if (reference !== null) throw new CreditsError('invalid_credits_request', 400); + if (unknown && (kind !== 'reserve' || amount_microusd !== 0)) throw new CreditsError('invalid_credits_request', 400); + return { owner, kind, amount_microusd, operation_id, revision, scoped_id, reference, tariff_revision, actor, reason, unknown }; +} +const documentFor = fields => JSON.stringify(Object.fromEntries(Object.entries(fields).sort(([a], [b]) => a.localeCompare(b)))); +const publicEntry = row => Object.fromEntries(entryKeys.map(key => [key, row[key]])); + +function deltas(fields, entries) { + const { kind, amount_microusd: amount } = fields; + let bd = 0, hd = 0; + if (['credit', 'adjust'].includes(kind)) bd = amount; + else if (kind === 'compensate') { + const original = entries.find(row => row.operation_id === fields.reference); + if (!original || !['credit', 'adjust', 'commit'].includes(original.kind) || amount !== -original.balance_delta || + entries.some(row => row.kind === 'compensate' && row.reference === fields.reference)) throw new CreditsError('credits_conflict', 409); + bd = amount; + } else { + const attempt = entries.filter(row => row.scoped_id === fields.scoped_id); + const hold = attempt.find(row => row.kind === 'reserve' && row.reference === null); + if (kind === 'reserve' && !fields.unknown) { + if (hold) throw new CreditsError('credits_conflict', 409); + hd = amount; + } else { + if (!hold || attempt.some(row => ['commit', 'release'].includes(row.kind))) throw new CreditsError('credits_conflict', 409); + fields.reference = hold.operation_id; + if (kind === 'commit') { bd = -amount; hd = -hold.amount_microusd; } + else if (kind === 'release') { + if (amount !== hold.amount_microusd) throw new CreditsError('invalid_credits_request', 400); + hd = -amount; + } + } + } + return [bd, hd]; +} + +async function schema(db) { + if (!db || typeof db.prepare !== 'function' || typeof db.batch !== 'function') throw new CreditsError(); + // Probe all required columns before any owner initialization or write. + await db.batch([ + db.prepare('SELECT owner,balance_microusd,held_microusd,revision FROM credits_balances LIMIT 0'), + db.prepare('SELECT owner,operation_id,revision,kind,amount_microusd,balance_delta,held_delta,scoped_id,reference,tariff_revision,actor,reason,document FROM credits_entries LIMIT 0'), + db.prepare('SELECT owner,operation_id,revision,actor,kind FROM credits_audit LIMIT 0'), + ]); +} + +export async function appendCredit(db, data) { + const fields = requestFields(data); + const document = documentFor(fields); + try { + await schema(db); + const find = () => db.prepare('SELECT * FROM credits_entries WHERE owner=? AND operation_id=?') + .bind(fields.owner, fields.operation_id).first(); + const replay = row => { + if (row.document !== document) throw new CreditsError('credits_conflict', 409); + return publicEntry(row); + }; + const existing = await find(); + if (existing) return replay(existing); + const row = await db.prepare('SELECT * FROM credits_balances WHERE owner=?').bind(fields.owner).first(); + const balance = row?.balance_microusd ?? 0, held = row?.held_microusd ?? 0, revision = row?.revision ?? 0; + if (revision !== fields.revision) { + const duplicate = await find(); + if (duplicate) return replay(duplicate); + throw new CreditsError('credits_revision_mismatch', 412); + } + const entries = (await db.prepare('SELECT * FROM credits_entries WHERE owner=? AND (scoped_id=? OR operation_id=? OR reference=?)') + .bind(fields.owner, fields.scoped_id, fields.reference, fields.reference).all()).results; + const [bd, hd] = deltas(fields, entries); + integer(balance + bd); integer(held + hd, true); integer(revision + 1, true); + const available = balance + bd - held - hd; + const reservation = fields.kind === 'reserve' && !fields.unknown; + const enforcedSpend = reservation || (fields.kind === 'adjust' && fields.amount_microusd < 0); + if (enforcedSpend && (available < 0 || (reservation && balance - held <= 0))) { + throw new CreditsError('credits_insufficient', 409); + } + const { owner, operation_id, kind, amount_microusd, scoped_id, reference, tariff_revision, actor, reason } = fields; + try { + await db.batch([ + db.prepare('INSERT OR IGNORE INTO credits_balances(owner) VALUES (?)').bind(owner), + db.prepare(`INSERT INTO credits_entries + (owner,operation_id,revision,kind,amount_microusd,balance_delta,held_delta,scoped_id,reference,tariff_revision,actor,reason,document) + SELECT ?,?,?,?,?,?,?,?,?,?,?,?,? FROM credits_balances WHERE owner=? AND revision=?`) + .bind(owner, operation_id, revision + 1, kind, amount_microusd, bd, hd, scoped_id, reference, + tariff_revision, actor, reason, document, owner, revision), + db.prepare(`UPDATE credits_balances SET balance_microusd=?,held_microusd=?,revision=revision+1 + WHERE owner=? AND revision=? AND EXISTS + (SELECT 1 FROM credits_entries WHERE owner=? AND operation_id=? AND revision=? AND document=?)`) + .bind(balance + bd, held + hd, owner, revision, owner, operation_id, revision + 1, document), + db.prepare(`INSERT OR IGNORE INTO credits_audit SELECT owner,operation_id,revision,actor,kind + FROM credits_entries WHERE owner=? AND operation_id=? AND actor IS NOT NULL`) + .bind(owner, operation_id), + ]); + } catch (error) { + const duplicate = await find(); + if (duplicate) return replay(duplicate); + throw error; + } + const saved = await find(); + if (!saved) throw new CreditsError('credits_revision_mismatch', 412); + return replay(saved); + } catch (error) { throw error instanceof CreditsError ? error : new CreditsError(); } +} + +export async function readCredits(db, owner, { cursor = '0', limit = 100 } = {}) { + ownerId(owner); + if (typeof cursor !== 'string' || !/^[0-9]{1,16}$/.test(cursor) || /\n|\r/.test(cursor) || + Number(cursor) > MAX_AMOUNT || !Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new CreditsError('invalid_credits_request', 400); + } + try { + await schema(db); + const results = await db.batch([ + db.prepare('SELECT * FROM credits_balances WHERE owner=?').bind(owner), + db.prepare('SELECT * FROM credits_entries WHERE owner=? AND revision>? ORDER BY revision LIMIT ?').bind(owner, Number(cursor), limit + 1), + ]); + const row = results[0].results[0], rows = results[1].results, page = rows.slice(0, limit); + const balance = row?.balance_microusd ?? 0, held = row?.held_microusd ?? 0; + return { currency: 'USD', balance_microusd: balance, held_microusd: held, available_microusd: balance - held, + revision: row?.revision ?? 0, entries: page.map(publicEntry), next_cursor: rows.length > limit ? String(page.at(-1).revision) : null }; + } catch (error) { throw error instanceof CreditsError ? error : new CreditsError(); } +} + +const reply = (body, status = 200) => Response.json({ version: 1, ...body }, { + status, headers: { 'Cache-Control': 'no-store' }, +}); + +async function boundedBody(request) { + const reader = request.body?.getReader(); + if (!reader) throw new CreditsError('invalid_credits_request', 400); + const parts = []; let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 16384) throw new CreditsError('invalid_credits_request', 400); + parts.push(value); + } + const bytes = new Uint8Array(size); let offset = 0; + for (const part of parts) { bytes.set(part, offset); offset += part.byteLength; } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + } catch (error) { throw error instanceof CreditsError ? error : new CreditsError('invalid_credits_request', 400); } + finally { void reader.cancel().catch(() => {}); } +} + +export async function handleCreditsRequest(request, env) { + const url = new URL(request.url); + if (url.origin !== 'http://intelligence.internal' || url.username || url.password || url.search || url.hash || + !['/v1/credits', '/v1/managed-state/credits'].includes(url.pathname) || !creditsEnabled(env)) { + return reply({ error: { code: 'not_found', message: 'Credits operation unavailable.' } }, 404); + } + try { + if (request.method !== 'POST') return reply({ error: { code: 'method_not_allowed' } }, 405); + const body = await boundedBody(request); + if (!body || typeof body !== 'object' || Array.isArray(body) || body.version !== 1 || !['append', 'read'].includes(body.operation)) { + throw new CreditsError('invalid_credits_request', 400); + } + const required = body.operation === 'append' ? ['owner', 'kind', 'amount_microusd', 'operation_id', 'revision'] : ['owner']; + const optional = body.operation === 'append' ? ['scoped_id', 'reference', 'tariff_revision', 'actor', 'reason', 'unknown'] : ['cursor', 'limit']; + if (required.some(key => !(key in body)) || Object.keys(body).some(key => !['version', 'operation', ...required, ...optional].includes(key))) { + throw new CreditsError('invalid_credits_request', 400); + } + const db = env.INTELLIGENCE_DB; + return body.operation === 'append' ? reply({ entry: await appendCredit(db, body) }) + : reply({ summary: await readCredits(db, body.owner, body) }); + } catch (error) { + const failure = error instanceof CreditsError ? error : new CreditsError(); + return reply({ error: { code: failure.code, message: 'Credits operation unavailable.' } }, failure.status); + } +} + +export function registerCreditAuthority(db, { tariff, env = {} }) { + if (typeof tariff !== 'function') throw new TypeError('Explicit operator tariff required'); + const apply = async (operation, phase) => { + if (!(operation instanceof AuthorityOperation)) throw new TypeError('AuthorityOperation required'); + if (!creditsEnabled(env)) throw new CreditDenied('credits_disabled', 404); + try { + const tariff_revision = await tariff(operation, phase); + const unknown = phase === 'reconcile' && tariff_revision === null; + if (tariff_revision === null && !unknown) throw new CreditDenied('credits_unpriced'); + const row = await appendCredit(db, { owner: contextOwner(operation.context), kind: phase === 'reserve' || unknown ? 'reserve' : 'commit', + amount_microusd: unknown ? 0 : operation.amount, operation_id: operation.operation_id, revision: operation.revision, + scoped_id: operation.scoped_id, tariff_revision, unknown }); + return new AuthorityResult({ context: operation.context, scoped_id: operation.scoped_id, + operation_id: operation.operation_id, revision: row.revision, allowed: true }); + } catch (error) { throw error instanceof CreditsError ? new CreditDenied(error.code, error.status) : error; } + }; + return Object.freeze({ reserve: op => apply(op, 'reserve'), commit: op => apply(op, 'commit'), reconcile: op => apply(op, 'reconcile') }); +} diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..a6b4cc3c 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "credits_balances", "credits_entries", "credits_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports From 649f8f9c136662981ba87e416bbd6362adc118a1 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:35:56 +0530 Subject: [PATCH 5/9] feat(governance): add opt-in hierarchical grants, budgets and attribution TENANT_GOVERNANCE_ENABLED (off by default) applies organisation and team policies to requests in a verified workspace. Model and tool grants intersect with the key's own grants and never widen them. Budgets at every configured level, plus the key's own limits, are reserved together in one atomic operation and settled together; an over-limit level denies the request before dispatch, and an unknown cost keeps its hold at every level. Roles come from a static permission table. Administrators edit policies through /admin/organisations/[/teams/]/governance with If-Match and audit rows, and /v1/usage gains a workspace breakdown for callers in a workspace. Requests outside a workspace are unchanged (migration 0034). --- docs/tenant-governance.md | 76 +++ .../0034_tenant_governance.sql | 30 ++ package.json | 2 +- routes/usage.py | 77 ++- services/budget_service.py | 18 +- services/intelligence_policy.py | 4 +- services/key_controls.py | 23 +- services/tenant_governance.py | 485 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 3 + tests/test_tenant_governance.py | 364 +++++++++++++ tests/test_tenant_governance_worker.mjs | 214 ++++++++ worker/container-env.mjs | 1 + worker/d1-schema.mjs | 2 +- worker/tenant-governance-d1.mjs | 315 ++++++++++++ 14 files changed, 1601 insertions(+), 13 deletions(-) create mode 100644 docs/tenant-governance.md create mode 100644 intelligence-migrations/0034_tenant_governance.sql create mode 100644 services/tenant_governance.py create mode 100644 tests/test_tenant_governance.py create mode 100644 tests/test_tenant_governance_worker.mjs create mode 100644 worker/tenant-governance-d1.mjs diff --git a/docs/tenant-governance.md b/docs/tenant-governance.md new file mode 100644 index 00000000..336658ae --- /dev/null +++ b/docs/tenant-governance.md @@ -0,0 +1,76 @@ +# Tenant governance + +`TENANT_GOVERNANCE_ENABLED` is off by default. Empty values are off. Invalid values +produce one warning without logging the value and are treated as off. When disabled, +key controls, budgets, routing and usage keep their existing behaviour; the governance +administration endpoints return 404. Legacy principals keep that behaviour when enabled. + +The server supplies a verified `TenantContext` through a registered `tenant_resolver` +and an organisation role through `membership_role(principal_id, org_id)`. The default +resolver uses the enterprise legacy authority; the default membership role grants no +permissions. Request headers and JSON fields cannot choose an organisation or role. + +## Grants and permissions + +The key grant, organisation grant and team grant must all allow a model or tool. +A missing ancestor row or a null grant adds no restriction. An empty array denies +everything at that level. Patterns are case insensitive; `*` matches any characters. +The same model intersection applies to automatic route candidates and model-free +provider calls. A provider call without a model requires a whole-provider grant. + +Organisation administrators can edit grants and budgets and read workspace usage. +Billing members can read budgets and workspace usage. Members can read their own +workspace usage. These endpoints cannot edit roles or memberships. + +`GET /admin/organisations//governance` and +`GET /admin/organisations//teams//governance` require an administrator +dashboard session and an appropriate organisation membership. Their `PUT` counterparts +also require CSRF and the quoted revision in `If-Match`. Missing revisions return 428; +stale revisions return 412. Responses expose the successor revision as `ETag`. + +Example policy body with governance enabled: + +```json +{"models":["openai:*"],"tools":["search"],"daily":1000000,"monthly":20000000} +``` + +Budget values are non-negative integer micro-USD, below 9,007,199,254,740,991. +`PUT` replaces the policy: omitted fields become null. It does not add to a grant. +Audits contain identifiers, revision, operation kind and time, without policy content. + +## Monetary reservations and usage + +An enabled workspace admission reserves the estimate against the key, organisation +and team together. Daily periods use UTC dates; monthly periods use UTC calendar +months. A zero budget denies admission. Existing key spending seeds its period +baseline. Unpriced requests return 503 `unpriced_reservation`. An ancestor limit +returns 429 `tenant_budget_exceeded` with the rejecting level; a key limit keeps +`budget_exceeded`. Rejected admission creates no reservation or component. + +Measured completion settles all components atomically. Unknown cost retains the +conservative estimate at every level, including across period changes; holds do +not expire automatically. Pre-dispatch cancellation can release a reservation. +Settlement is idempotent and cannot replace a settled cost with a different amount. +Actual cost can exceed the admission estimate; configure conservative pricing and +output bounds. Estimates and measured gateway costs are not provider invoices. + +`GET /v1/usage` includes `workspace` only for enabled workspace callers. Its history, +model totals and daily totals use that verified workspace, without reading unscoped +legacy history. Members see their own records; administrators and billing members +see workspace totals plus organisation and selected-team totals. Model rows identify +the actual model, provider and price basis. Results have at most 200 model groups +and 90 daily groups. Usage and audits contain no prompts or response content. + +Apply `0034_tenant_governance.sql` before enabling governance. The migration creates +only new tables and indexes and can be rehearsed against existing rows. A missing +table, column or unavailable authority returns JSON 503 +`tenant_governance_unavailable` before admission. No permissive fallback is used. + +The D1 domain is mounted only behind the authenticated private dispatcher at +`http://intelligence.internal/v1/tenant-governance`. Native admission uses +`createGovernanceLifecycle` with verified tenant and key decisions, integer price +estimates, dispatch and finalization hooks. Flask uses the same domain through +`D1GovernanceStore`; `SQLiteGovernanceStore` supports explicitly injected connection +factories and database transactions. Neither adapter applies migrations automatically. +Storage failure prevents new admissions; ambiguous settlement leaves the hold in place. +Retention and any reconciliation of unknown holds require an operator policy. diff --git a/intelligence-migrations/0034_tenant_governance.sql b/intelligence-migrations/0034_tenant_governance.sql new file mode 100644 index 00000000..9afd1048 --- /dev/null +++ b/intelligence-migrations/0034_tenant_governance.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS tenant_governance_policies ( + org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', revision INTEGER NOT NULL DEFAULT 0, + models TEXT, tools TEXT, daily INTEGER CHECK (daily IS NULL OR daily >= 0), + monthly INTEGER CHECK (monthly IS NULL OR monthly >= 0), PRIMARY KEY (org_id, team_id) +); +CREATE TABLE IF NOT EXISTS tenant_governance_reservations ( + id TEXT PRIMARY KEY, principal_id TEXT NOT NULL, org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', + day TEXT NOT NULL, month TEXT NOT NULL, estimate INTEGER NOT NULL CHECK (estimate >= 0), + charged INTEGER, state TEXT NOT NULL CHECK (state IN ('reserved','dispatched','unknown','settled')), + revision INTEGER NOT NULL DEFAULT 0, provider TEXT, model TEXT, price_basis TEXT, + created_at TEXT NOT NULL, operation_id TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS tenant_governance_components ( + reservation_id TEXT NOT NULL REFERENCES tenant_governance_reservations(id), + level TEXT NOT NULL, scope_org TEXT NOT NULL, scope_id TEXT NOT NULL, + period_kind TEXT NOT NULL, period TEXT NOT NULL, limit_units INTEGER, + PRIMARY KEY (reservation_id, level, scope_org, scope_id, period_kind) +); +CREATE INDEX IF NOT EXISTS tenant_governance_component_scope + ON tenant_governance_components(level, scope_org, scope_id, period_kind, period); +CREATE INDEX IF NOT EXISTS tenant_governance_usage_scope + ON tenant_governance_reservations(org_id, team_id, principal_id, day); +CREATE TABLE IF NOT EXISTS tenant_governance_baselines ( + principal_id TEXT NOT NULL, period TEXT NOT NULL, amount INTEGER NOT NULL, + PRIMARY KEY (principal_id, period) +); +CREATE TABLE IF NOT EXISTS tenant_governance_audit ( + operation_id TEXT PRIMARY KEY, actor TEXT NOT NULL, org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', + revision INTEGER NOT NULL, kind TEXT NOT NULL, at TEXT NOT NULL +); diff --git a/package.json b/package.json index c66e4a8f..934ee52b 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_tenant_governance_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/usage.py b/routes/usage.py index 66b99489..8cd57ab3 100644 --- a/routes/usage.py +++ b/routes/usage.py @@ -2,6 +2,7 @@ administrator's per-key controls.""" import logging +import re from datetime import datetime, timedelta, timezone from flask import g, jsonify, render_template, request @@ -10,7 +11,7 @@ from middleware.rate_limit_headers import enabled as rate_limit_headers_enabled, usage_snapshot from request_validation import json_object_body from route_helpers import api_authenticate_only, login_required -from services import key_controls, usage_ledger, usage_store +from services import key_controls, usage_ledger, usage_store, tenant_governance from services.auth_service import AuthService from services.budget_service import BudgetService @@ -65,6 +66,7 @@ def usage_history(principal, days: int, *, by_principal: bool = False) -> dict: def register_usage_routes(app, csrf) -> None: + register_governance_routes(app) @app.route("/v1/usage", methods=["GET", "OPTIONS"]) @csrf.exempt @api_authenticate_only(required_scope="models") @@ -73,6 +75,16 @@ def api_usage(): user = g.authenticated_user principal = str(user.get("username") or user.get("id")) controls = key_controls.public(user) + since, until = _range(_days()) + workspace = tenant_governance.workspace_usage(user, since=since, until=until) + if workspace: + scoped = workspace["workspace"] + return jsonify({"object": "usage", "principal": principal, "key_prefix": user.get("api_key_prefix"), + "controls": {name: controls[name] for name in ("allowed_models", "allowed_ips", "expires_at")}, + "budget": {"daily_budget_usd": user.get("daily_budget_usd"), + "monthly_budget_usd": user.get("monthly_budget_usd")}, + "history_available": True, "range": scoped["range"], "daily": scoped["daily"], "models": scoped["models"], + "totals": {key: scoped[key] for key in ("spent_micro_usd", "holds_micro_usd", "requests")}, **workspace}) return jsonify({ **({"rate_limits": usage_snapshot(user, request.remote_addr)} if rate_limit_headers_enabled() else {}), @@ -82,6 +94,7 @@ def api_usage(): "budget": BudgetService.status(user), "controls": {name: controls[name] for name in ("allowed_models", "allowed_ips", "expires_at")}, **usage_history(principal, _days()), + **workspace, }) @app.route("/usage") @@ -123,3 +136,65 @@ def update_key_controls(username: str): except APIError as error: return jsonify({"status": "error", "message": error.client_message}), error.status_code return jsonify({"status": "success", "user": user}) + + +def register_governance_routes(app): + if app.extensions.get("tenant_governance_routes_registered"): + return + app.extensions["tenant_governance_routes_registered"] = True + + def disabled_governance(): + if request.path.startswith("/admin/organisations/") and request.path.endswith("/governance"): + if not tenant_governance.enabled(): + return jsonify({"error": {"code": "not_found"}}), 404 + return None + + app.before_request_funcs.setdefault(None, []).insert(0, disabled_governance) + + @app.after_request + def governance_budget_envelope(response): + if tenant_governance.enabled() and response.status_code == 429 and response.is_json: + payload = response.get_json(silent=True) + if isinstance(payload, dict) and payload.get("error") == "tenant_budget_exceeded" and isinstance(payload.get("budget"), dict): + payload["error"] = {"code": "tenant_budget_exceeded", "level": payload["budget"].get("level")} + response.set_data(app.json.dumps(payload) + "\n") + return response + + @app.errorhandler(tenant_governance.GovernanceError) + def governance_error(error): + return jsonify(error.payload), error.status + + def governance(org_id, team_id=None): + user = AuthService.get_current_user() or {} + if not user.get("is_admin"): + raise tenant_governance.GovernanceError("administrator_required", 403) + tenant_governance._scope(org_id, team_id) + principal = str(user.get("username") or user.get("id") or "") + role = tenant_governance.membership_role(principal, org_id) + permission = "edit" if request.method == "PUT" else "budgets" + if not tenant_governance.permitted(role, permission): + raise tenant_governance.GovernanceError("tenant_role_denied", 403) + values = {"org_id": org_id, "team_id": team_id} + if request.method == "PUT": + match = request.headers.get("If-Match") + if match is None: + raise tenant_governance.GovernanceError("revision_required", 428) + if not re.fullmatch(r'"(?:0|[1-9][0-9]{0,15})"', match): + raise tenant_governance.GovernanceError("revision_stale", 412) + values.update(actor=principal, revision=int(match[1:-1]), + policy=tenant_governance.validate_policy(json_object_body())) + result = tenant_governance.store().call("put" if request.method == "PUT" else "get", **values) + response = jsonify(result) + response.headers["ETag"] = f'"{result["policy"]["revision"]}"' + response.headers["Cache-Control"] = "no-store" + return response + + @app.route("/admin/organisations//governance", methods=["GET", "PUT"]) + @login_required + def organisation_governance(org_id): + return governance(org_id) + + @app.route("/admin/organisations//teams//governance", methods=["GET", "PUT"]) + @login_required + def team_governance(org_id, team_id): + return governance(org_id, team_id) diff --git a/services/budget_service.py b/services/budget_service.py index 4011fe32..022aa65a 100644 --- a/services/budget_service.py +++ b/services/budget_service.py @@ -55,7 +55,8 @@ def limits(user: Mapping[str, Any]) -> tuple[Optional[float], Optional[float]]: def budgeted(user: Mapping[str, Any]) -> bool: - return any(limit is not None for limit in limits(user)) + from services.tenant_governance import workspace_context + return workspace_context(user) is not None or any(limit is not None for limit in limits(user)) @dataclass @@ -201,6 +202,10 @@ def _spent(cls, state: _Principal, period: dict) -> tuple[float, float, float]: @classmethod def check_and_reserve(cls, user: Mapping[str, Any], estimate_usd: Optional[float], now: Optional[datetime] = None) -> BudgetDecision: + from services.tenant_governance import reserve_budget + governance = reserve_budget(user, estimate_usd, now, cls) + if governance is not None: + return governance daily, monthly = limits(user) if daily is None and monthly is None: return BudgetDecision(True) @@ -278,6 +283,9 @@ def _reserve_durable(cls, user, estimate, now=None): @classmethod def mark_dispatched(cls, reservation): """Persist before provider submission; failure must prevent submission.""" + from services.tenant_governance import dispatch + if dispatch(reservation): + return if not reservation or not reservation_store.enabled(): return store = reservation_store.get_store() @@ -290,6 +298,9 @@ def mark_dispatched(cls, reservation): @classmethod def complete(cls, reservation, row, *, cached=False): """Replace a hold only with measured cost; uncertain completion retains it.""" + from services.tenant_governance import complete + if complete(reservation, row, cached=cached): + return if not reservation or not reservation_store.enabled(): return store = reservation_store.get_store() @@ -314,6 +325,9 @@ def settle(cls, reservation: Optional[str], *, before_dispatch=False) -> None: """Release an in-flight estimate; the settled cost arrives through `record_cost`.""" if not reservation: return + from services.tenant_governance import complete + if complete(reservation, {}, before_dispatch=before_dispatch): + return if reservation_store.enabled(): if before_dispatch: store = reservation_store.get_store() @@ -336,6 +350,8 @@ def record_cost(cls, row: Mapping[str, Any]) -> None: Call before handing the row to the ledger, which then reports it as stored or dropped exactly once. """ + from services.tenant_governance import capture_cost + capture_cost(row) with cls._lock: state = cls._principals.setdefault(str(row["principal"]), _Principal()) _add(state.pending, row) diff --git a/services/intelligence_policy.py b/services/intelligence_policy.py index 174e2726..cec1fbab 100644 --- a/services/intelligence_policy.py +++ b/services/intelligence_policy.py @@ -189,12 +189,14 @@ def validate_policy(raw): def eligible(candidate, allow_paid, config, *, model_status=None): provider = candidate["model"].split(":", 1)[0] model_status = model_status or ModelRegistry.get_model_status - return ( + allowed = ( all(candidate[key] for key in ("enabled", "entitled", "privacy_allowed")) and (candidate["billing"] != "payg" or allow_paid) and model_status(candidate["model"]) != "disabled" and get_adapter(provider, config["API_BASE_URLS"]) is not None ) + from services.tenant_governance import eligibility_allowed + return eligibility_allowed(allowed, candidate["model"]) def input_reservation(candidate, request): diff --git a/services/key_controls.py b/services/key_controls.py index 633db632..017ee8b0 100644 --- a/services/key_controls.py +++ b/services/key_controls.py @@ -212,20 +212,27 @@ def model_allowed(user: Mapping[str, Any], model: Any) -> bool: """Whether a model ID matches the key's allowlist; ``*`` matches any characters.""" patterns = model_patterns(user) if patterns is None: - return True - if not isinstance(model, str) or not model.strip(): - return False - candidate = model.strip().lower() - return any(_glob(pattern.lower()).match(candidate) for pattern in patterns) + allowed = True + elif not isinstance(model, str) or not model.strip(): + allowed = False + else: + candidate = model.strip().lower() + allowed = any(_glob(pattern.lower()).match(candidate) for pattern in patterns) + from services.tenant_governance import grant_allowed + return grant_allowed(allowed, model, user=user) def provider_allowed(user: Mapping[str, Any], provider: str) -> bool: """Whether a request that names no model may reach a provider namespace.""" patterns = model_patterns(user) if patterns is None: - return True - provider = provider.strip().lower() - return any(pattern in {"*", f"{provider}:*"} for pattern in patterns) + allowed = True + candidate = provider.strip().lower() if isinstance(provider, str) else "" + else: + candidate = provider.strip().lower() + allowed = any(pattern in {"*", f"{candidate}:*"} for pattern in patterns) + from services.tenant_governance import grant_allowed + return grant_allowed(allowed, f"{candidate}:*", user=user) def expires_at(user: Mapping[str, Any]) -> Optional[datetime]: diff --git a/services/tenant_governance.py b/services/tenant_governance.py new file mode 100644 index 00000000..3b128c75 --- /dev/null +++ b/services/tenant_governance.py @@ -0,0 +1,485 @@ +"""Intersected workspace grants and durable multi-level monetary reservations.""" +from __future__ import annotations + +import json +import logging +import os +import re +import sqlite3 +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from decimal import Decimal, ROUND_CEILING +from functools import lru_cache + +from flask import current_app, g, has_app_context, has_request_context + +from error_handlers import APIError +from services import control_state_d1 +from services.enterprise_contract import AuthorityOperation, MAX_INTEGER, TenantContext, legacy_tenant + +logger = logging.getLogger(__name__) +PERMISSIONS = { + "admin": frozenset({"edit", "budgets", "usage", "all_usage"}), + "billing": frozenset({"budgets", "usage", "all_usage"}), + "member": frozenset({"usage"}), +} +TABLES = ("tenant_governance_policies", "tenant_governance_reservations", + "tenant_governance_components", "tenant_governance_baselines", "tenant_governance_audit") +SCHEMA_QUERIES = ( + "SELECT org_id,team_id,revision,models,tools,daily,monthly FROM tenant_governance_policies LIMIT 0", + "SELECT id,principal_id,org_id,team_id,day,month,estimate,charged,state,revision,provider,model,price_basis,created_at,operation_id FROM tenant_governance_reservations LIMIT 0", + "SELECT reservation_id,level,scope_org,scope_id,period_kind,period,limit_units FROM tenant_governance_components LIMIT 0", + "SELECT principal_id,period,amount FROM tenant_governance_baselines LIMIT 0", + "SELECT operation_id,actor,org_id,team_id,revision,kind,at FROM tenant_governance_audit LIMIT 0", +) +POLICY_FIELDS = frozenset({"models", "tools", "daily", "monthly"}) + + +class GovernanceError(APIError): + def __init__(self, code="tenant_governance_unavailable", status=503, *, level=None, period=None): + self.code, self.status, self.level, self.period = code, status, level, period + detail = {"code": code} + if level: + detail["level"] = level + if period: + detail["period"] = period + super().__init__("The workspace governance operation could not be completed.", status, {"error": detail}) + + +@lru_cache(maxsize=1) +def _warn_once(): + logger.warning("Invalid TENANT_GOVERNANCE_ENABLED; tenant governance disabled") + + +def enabled(env=None): + value = (os.environ if env is None else env).get("TENANT_GOVERNANCE_ENABLED", "") + flag = value.strip().lower() if isinstance(value, str) else "invalid" + if flag in {"", "false", "0", "off", "no"}: + return False + if flag in {"true", "1", "on", "yes"}: + return True + _warn_once() + return False + + +def permitted(role, permission): + return permission in PERMISSIONS.get(role, ()) + + +def _legacy_resolver(user): + context = TenantContext(str(user.get("username") or user.get("id") or "")) + return legacy_tenant(AuthorityOperation(context, context.principal_id, 0, "governance.resolve")) + + +def _no_role(principal_id, org_id): + return None + + +@dataclass(frozen=True) +class GovernanceCollaborators: + tenant_resolver: object = _legacy_resolver + membership_role: object = _no_role + store: object = None + + +_collaborators = GovernanceCollaborators() + + +def register_governance_collaborators(*, tenant_resolver=_legacy_resolver, + membership_role=_no_role, store=None, app=None): + """Register fixed trusted collaborators; caller metadata never selects a workspace.""" + if not callable(tenant_resolver) or not callable(membership_role): + raise TypeError("Named tenant resolver and membership role functions required") + if store is not None and not callable(getattr(store, "call", None)): + raise TypeError("Governance store call interface required") + collaborators = GovernanceCollaborators(tenant_resolver, membership_role, store) + if app is not None: + app.extensions["tenant_governance"] = collaborators + else: + global _collaborators + _collaborators = collaborators + return collaborators + + +def collaborators(): + return current_app.extensions.get("tenant_governance", _collaborators) if has_app_context() else _collaborators + + +def workspace_context(user=None): + if not enabled(): + return None + user = user if user is not None else (getattr(g, "authenticated_user", None) if has_request_context() else None) + if user is None: + return None + try: + context = collaborators().tenant_resolver(user) + if type(context) is not TenantContext or context.principal_id != str(user.get("username") or user.get("id") or ""): + raise GovernanceError("tenant_scope_denied", 403) + except GovernanceError: + raise + except Exception: + raise GovernanceError("tenant_scope_denied", 403) from None + return context if context.org_id is not None else None + + +def context_dict(context): + return asdict(context) + + +def membership_role(principal_id, org_id): + return collaborators().membership_role(principal_id, org_id) + + +class D1GovernanceStore: + def call(self, operation, **values): + if not control_state_d1.using_d1(): + raise GovernanceError() + try: + result = control_state_d1.call("tenant_governance", operation, version=1, rpc=True, **values) + except control_state_d1.intelligence_d1_store.PrivateIntelligenceError as error: + raise GovernanceError(error.code, error.status) from None + except Exception: + raise GovernanceError() from None + if not isinstance(result, dict) or result.get("version") != 1: + raise GovernanceError() + decision = result.get("decision") + if isinstance(decision, dict) and decision.get("allowed") is False: + raise GovernanceError(decision["code"], decision["status"], + level=decision.get("level"), period=decision.get("period")) + if result.get("error"): + failure = result["error"] + code = failure.get("code", "tenant_governance_unavailable") + status = {"tenant_budget_exceeded": 429, "budget_exceeded": 429, + "revision_stale": 412, "reservation_conflict": 409}.get(code, 503) + raise GovernanceError(code, status, level=failure.get("level"), period=failure.get("period")) + return {key: value for key, value in result.items() if key != "version"} + + +def store(): + return collaborators().store or D1GovernanceStore() + + +def validate_policy(value): + if not isinstance(value, dict) or set(value) - POLICY_FIELDS: + raise GovernanceError("invalid_governance_policy", 400) + result = {key: value.get(key) for key in POLICY_FIELDS} + for key in ("daily", "monthly"): + amount = result[key] + if amount is not None and (type(amount) is not int or not 0 <= amount < MAX_INTEGER): + raise GovernanceError("invalid_governance_policy", 400) + for key in ("models", "tools"): + entries = result[key] + if entries is not None and (not isinstance(entries, list) or len(entries) > 64 or + any(not isinstance(entry, str) or re.fullmatch(r"[A-Za-z0-9._:/+@*\-]{1,256}", entry) is None for entry in entries)): + raise GovernanceError("invalid_governance_policy", 400) + return result + + +def _matches(patterns, candidate): + if patterns is None: + return True + if not isinstance(candidate, str) or not candidate.strip(): + return False + return any(re.fullmatch(".*".join(re.escape(part) for part in pattern.lower().split("*")), + candidate.strip().lower()) for pattern in patterns) + + +def grant_allowed(existing_allowed, candidate, *, user=None, kind="models", context=None): + """Every configured ancestor intersects the existing key decision.""" + if not enabled(): + return existing_allowed + context = context if context is not None else workspace_context(user) + if context is None or context.org_id is None: + return existing_allowed + if kind not in {"models", "tools"}: + raise ValueError("Unknown grant kind") + policies = store().call("policies", context=context_dict(context))["policies"] + return existing_allowed and all(_matches(policy[kind], candidate) for policy in policies) + + +def eligibility_allowed(existing_allowed, model): + if not enabled() or not has_request_context(): + return existing_allowed + user = getattr(g, "authenticated_user", None) + if workspace_context(user) is None: + return existing_allowed + from services.key_controls import model_allowed + key_allowed = model_allowed(user, model) + return existing_allowed and key_allowed + + +def units(value): + if isinstance(value, bool) or value is None: + raise GovernanceError("unpriced_reservation", 503) + try: + number = Decimal(str(value)) * 1_000_000 + if not number.is_finite() or number < 0 or number >= MAX_INTEGER: + raise ValueError + return int(number.to_integral_value(rounding=ROUND_CEILING)) + except (ValueError, ArithmeticError): + raise GovernanceError("unpriced_reservation", 503) from None + + +def reserve_budget(user, estimate_usd, now, budget_service): + from services.budget_service import BudgetDecision, BudgetUnavailable, limits, periods + try: + context = workspace_context(user) + if context is None: + return None + amount = units(estimate_usd) + period = periods(now) + daily, monthly = limits(user) + base_day = base_month = 0 + if daily is not None or monthly is not None: + state = budget_service._state(context.principal_id) + budget_service._refresh(context.principal_id, state, period) + with budget_service._lock: + day, month, inflight = budget_service._spent(state, period) + base_day, base_month = units(day + inflight), units(month + inflight) + identity = "tg_" + uuid.uuid4().hex + store().call("reserve", id=identity, context=context_dict(context), amount=amount, + day=period["day"], month=period["month"], + key_daily=None if daily is None else units(daily), + key_monthly=None if monthly is None else units(monthly), + base_day=base_day, base_month=base_month) + if has_request_context(): + # Accounting clears usage_context before recording its final row. + g.tenant_governance_reservation = identity + g.tenant_governance_principal = context.principal_id + return BudgetDecision(True, reservation=identity) + except (GovernanceError, BudgetUnavailable) as error: + if isinstance(error, BudgetUnavailable): + error = GovernanceError() + details = {"level": error.level, "period": error.period} if error.level else {} + return BudgetDecision(False, error=error.code, status_code=error.status, + message="The workspace budget could not admit this request.", details=details) + + +def governance_reservation(identity): + return isinstance(identity, str) and identity.startswith("tg_") + + +def dispatch(identity): + if not governance_reservation(identity): + return False + store().call("dispatch", id=identity) + return True + + +def complete(identity, row, *, before_dispatch=False, cached=False): + if not governance_reservation(identity): + return False + model = row.get("selected_model") + model = model if isinstance(model, str) and re.fullmatch(r"[A-Za-z0-9._:/+@\-]{1,256}", model) else None + cost = row.get("cost_usd") + measured = cost is not None and row.get("cost_basis") == "usage" and not cached + store().call("settle", id=identity, cost=0 if before_dispatch or cached else units(cost) if measured else None, + provider=model.split(":", 1)[0] if model and ":" in model else None, model=model, + price_basis="released" if before_dispatch else "cache" if cached else row.get("cost_basis"), + before_dispatch=before_dispatch) + return True + + +def capture_cost(row): + if not has_request_context(): + return + identity = getattr(g, "tenant_governance_reservation", None) + if governance_reservation(identity) and row.get("principal") == getattr(g, "tenant_governance_principal", None): + complete(identity, row, cached=row.get("cost_basis") == "cache") + + +def workspace_usage(user, *, since="0000-01-01", until="9999-12-31"): + context = workspace_context(user) + if context is None: + return {} + role = membership_role(context.principal_id, context.org_id) + if not permitted(role, "usage"): + raise GovernanceError("tenant_role_denied", 403) + return {"workspace": store().call("usage", context=context_dict(context), role=role, since=since, until=until)} + + +def _scope(org_id, team_id): + try: + TenantContext("scope", org_id, team_id) + except (ValueError, TypeError): + raise GovernanceError("invalid_tenant_scope", 400) from None + if org_id is None: + raise GovernanceError("invalid_tenant_scope", 400) + return org_id, team_id or "" + + +def _policy(row): + result = {"revision": row["revision"], "models": json.loads(row["models"]) if row["models"] is not None else None, + "tools": json.loads(row["tools"]) if row["tools"] is not None else None, + "daily": row["daily"], "monthly": row["monthly"]} + try: + validate_policy({key: result[key] for key in POLICY_FIELDS}) + except GovernanceError: + raise GovernanceError() from None + return result + + +class SQLiteGovernanceStore: + """Injected SQLite adapter. Migration application belongs to the operator.""" + def __init__(self, connect): + self.connect = connect + + @contextmanager + def transaction(self): + db = self.connect() + db.row_factory = sqlite3.Row + try: + db.execute("BEGIN IMMEDIATE") + for query in SCHEMA_QUERIES: + db.execute(query) + yield db + db.commit() + except GovernanceError: + db.rollback() + raise + except (sqlite3.Error, OSError, ValueError, TypeError, KeyError): + db.rollback() + raise GovernanceError() from None + finally: + db.close() + + def call(self, operation, **values): + with self.transaction() as db: + if operation in {"get", "put"}: + return self._admin(db, operation, values) + if operation == "policies": + context = TenantContext(**values["context"]) + return {"policies": self._policies(db, context)} + if operation == "reserve": + return self._reserve(db, values) + if operation in {"settle", "dispatch"}: + return self._settle(db, operation, values) + if operation == "usage": + return self._usage(db, values) + raise GovernanceError("invalid_governance_operation", 400) + + def _policies(self, db, context): + return [_policy(row) for row in db.execute( + "SELECT * FROM tenant_governance_policies WHERE org_id=? AND (team_id='' OR team_id=?)", + (context.org_id, context.team_id or ""))] + + def _admin(self, db, operation, values): + org, team = _scope(values["org_id"], values.get("team_id")) + row = db.execute("SELECT * FROM tenant_governance_policies WHERE org_id=? AND team_id=?", (org, team)).fetchone() + revision = row["revision"] if row else 0 + if operation == "get": + return {"policy": _policy(row) if row else {"revision": 0, **dict.fromkeys(POLICY_FIELDS)}} + policy = validate_policy(values["policy"]) + if type(values["revision"]) is not int or values["revision"] != revision: + raise GovernanceError("revision_stale", 412) + actor = TenantContext(values["actor"]).principal_id + db.execute("""INSERT INTO tenant_governance_policies VALUES (?,?,?,?,?,?,?) + ON CONFLICT(org_id,team_id) DO UPDATE SET revision=excluded.revision,models=excluded.models, + tools=excluded.tools,daily=excluded.daily,monthly=excluded.monthly""", + (org, team, revision + 1, None if policy["models"] is None else json.dumps(policy["models"]), + None if policy["tools"] is None else json.dumps(policy["tools"]), policy["daily"], policy["monthly"])) + db.execute("INSERT INTO tenant_governance_audit VALUES (?,?,?,?,?,?,?)", + (uuid.uuid4().hex, actor, org, team, revision + 1, "policy", datetime.now(timezone.utc).isoformat())) + return {"policy": {"revision": revision + 1, **policy}} + + def _components(self, db, context, values): + components = [] + scopes = [("key", "", context.principal_id, values["key_daily"], values["key_monthly"])] + for team in [""] + ([context.team_id] if context.team_id else []): + row = db.execute("SELECT daily,monthly FROM tenant_governance_policies WHERE org_id=? AND team_id=?", + (context.org_id, team)).fetchone() + scopes.append(("team" if team else "organisation", context.org_id, team or context.org_id, + row["daily"] if row else None, row["monthly"] if row else None)) + for level, org, scoped, daily, monthly in scopes: + for kind, period, limit in (("daily", values["day"], daily), ("monthly", values["month"], monthly)): + components.append((level, org, scoped, kind, period, limit)) + return components + + def _reserve(self, db, values): + context = TenantContext(**values["context"]) + if context.org_id is None: + raise GovernanceError("tenant_scope_denied", 403) + amount = values["amount"] + if type(amount) is not int or not 0 <= amount < MAX_INTEGER: + raise GovernanceError("unpriced_reservation", 503) + if db.execute("SELECT id FROM tenant_governance_reservations WHERE id=?", (values["id"],)).fetchone(): + raise GovernanceError("reservation_conflict", 409) + components = self._components(db, context, values) + for period, baseline in ((values["day"], values["base_day"]), (values["month"], values["base_month"])): + db.execute("INSERT OR IGNORE INTO tenant_governance_baselines VALUES (?,?,?)", (context.principal_id, period, baseline)) + for level, org, scoped, kind, period, limit in components: + if limit is None: + continue + total = db.execute("""SELECT COALESCE(SUM(CASE WHEN r.state!='settled' THEN r.estimate + WHEN c.period=? THEN r.charged ELSE 0 END),0) FROM tenant_governance_components c + JOIN tenant_governance_reservations r ON r.id=c.reservation_id + WHERE c.level=? AND c.scope_org=? AND c.scope_id=? AND c.period_kind=?""", + (period, level, org, scoped, kind)).fetchone()[0] + if level == "key": + total += db.execute("SELECT amount FROM tenant_governance_baselines WHERE principal_id=? AND period=?", + (context.principal_id, period)).fetchone()[0] + if total >= limit or amount > limit - total: + raise GovernanceError("budget_exceeded" if level == "key" else "tenant_budget_exceeded", 429, + level=level, period=kind) + db.execute("""INSERT INTO tenant_governance_reservations + (id,principal_id,org_id,team_id,day,month,estimate,state,created_at,operation_id) VALUES (?,?,?,?,?,?,?,'reserved',?,?)""", + (values["id"], context.principal_id, context.org_id, context.team_id or "", values["day"], values["month"], + amount, datetime.now(timezone.utc).isoformat(), uuid.uuid4().hex)) + db.executemany("INSERT INTO tenant_governance_components VALUES (?,?,?,?,?,?,?)", + [(values["id"], *component) for component in components]) + return {"id": values["id"]} + + def _settle(self, db, operation, values): + row = db.execute("SELECT * FROM tenant_governance_reservations WHERE id=?", (values["id"],)).fetchone() + if row is None: + raise GovernanceError("reservation_conflict", 409) + if row["state"] in {"settled", "unknown"}: + if (row["state"] == "settled" and operation == "settle" and values.get("cost") is not None + and row["charged"] != values["cost"]): + raise GovernanceError("reservation_conflict", 409) + return {"state": row["state"]} + if operation == "dispatch": + db.execute("UPDATE tenant_governance_reservations SET state='dispatched',revision=revision+1 WHERE id=?", (values["id"],)) + return {"state": "dispatched"} + if values.get("before_dispatch") and row["state"] != "reserved": + raise GovernanceError("reservation_conflict", 409) + cost = values["cost"] + if cost is not None and (type(cost) is not int or not 0 <= cost < MAX_INTEGER): + raise GovernanceError("invalid_governance_cost", 400) + state = "unknown" if cost is None else "settled" + db.execute("""UPDATE tenant_governance_reservations SET state=?,charged=?,provider=?,model=?,price_basis=?, + revision=revision+1 WHERE id=?""", (state, cost, values.get("provider"), values.get("model"), + values.get("price_basis"), values["id"])) + return {"state": state} + + def _usage(self, db, values): + context = TenantContext(**values["context"]) + role = values["role"] + if not permitted(role, "usage"): + raise GovernanceError("tenant_role_denied", 403) + since, until = values.get("since", "0000-01-01"), values.get("until", "9999-12-31") + params = (context.org_id, context.team_id or "", context.team_id or "", + int(permitted(role, "all_usage")), context.principal_id, since, until) + totals = dict(db.execute("""SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd, + COUNT(*) AS requests FROM tenant_governance_reservations WHERE org_id=? AND (?='' OR team_id=?) + AND (?=1 OR principal_id=?) AND day>=? AND day<=?""", params).fetchone()) + models = [dict(row) for row in db.execute("""SELECT provider,model,price_basis,COUNT(*) AS requests, + COALESCE(SUM(charged),0) AS spent_micro_usd FROM tenant_governance_reservations + WHERE org_id=? AND (?='' OR team_id=?) AND (?=1 OR principal_id=?) AND day>=? AND day<=? + GROUP BY provider,model,price_basis ORDER BY model LIMIT 200""", params)] + daily = [dict(row) for row in db.execute("""SELECT day,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND (?='' OR team_id=?) + AND (?=1 OR principal_id=?) AND day>=? AND day<=? GROUP BY day ORDER BY day LIMIT 90""", params)] + result = {"org_id": context.org_id, "team_id": context.team_id, "principal_id": context.principal_id, + "cost_description": "Gateway cost estimates, not invoices", **totals, "models": models, "daily": daily, + "range": {"since": since, "until": until}} + if permitted(role, "all_usage"): + org_total = dict(db.execute("""SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND day>=? AND day<=?""", (context.org_id, since, until)).fetchone()) + result["organisation"] = org_total + result["team"] = totals if context.team_id else None + return result diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..0cbf8396 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -192,6 +192,8 @@ "app.route('/v1/messages/count_tokens', methods=['POST', 'OPTIONS'])" ], "routes/usage.py": [ + "app.route('/admin/organisations//governance', methods=['GET', 'PUT'])", + "app.route('/admin/organisations//teams//governance', methods=['GET', 'PUT'])", "app.route('/usage')", "app.route('/usage/data')", "app.route('/users//controls', methods=['PUT'])", @@ -409,6 +411,7 @@ "worker/sse-heartbeat.mjs", "worker/status-page.mjs", "worker/stream-cost-breaker.mjs", + "worker/tenant-governance-d1.mjs", "worker/upstream-cancellation.mjs", "worker/usage-buckets-d1.mjs", "worker/usage-ledger-d1.mjs", diff --git a/tests/test_tenant_governance.py b/tests/test_tenant_governance.py new file mode 100644 index 00000000..45ba8890 --- /dev/null +++ b/tests/test_tenant_governance.py @@ -0,0 +1,364 @@ +"""Governance policy, transactional quotas and registered HTTP boundaries.""" +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask, g +from flask_wtf import CSRFProtect +from flask_wtf.csrf import generate_csrf + +from services import tenant_governance as governance, key_controls, budget_service +from services.enterprise_contract import TenantContext +from tests.unified_api_test_case import UnifiedApiTestCase + +NOW = datetime(2026, 10, 9, tzinfo=timezone.utc) +USER = {"username": "alice", "daily_budget_usd": 1.0, "allowed_models": "openai:*"} +TENANT = TenantContext("alice", "org1", "team1") + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + monkeypatch.setenv("USAGE_RESERVATIONS_ENABLED", "false") + monkeypatch.setenv("USAGE_LEDGER_ENABLED", "false") + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + store = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path, timeout=10)) + app = Flask(__name__) + app.secret_key = "synthetic-governance-test-secret" + governance.register_governance_collaborators(app=app, store=store, + tenant_resolver=lambda user: TenantContext(user["username"], "org1", "team1"), + membership_role=lambda principal, org: "admin" if org == "org1" else None) + budget_service.BudgetService.reset() + yield app, store + budget_service.BudgetService.reset() + + +def policy(store, org="org1", team=None, **fields): + return store.call("put", org_id=org, team_id=team, actor="alice", revision=0, + policy={"models": None, "tools": None, "daily": None, "monthly": None, **fields}) + + +@pytest.mark.parametrize("flag", [None, "", "false", "broken"]) +def test_disabled_never_resolves_or_reads_storage(setup, monkeypatch, flag): + app, _ = setup + if flag is None: + monkeypatch.delenv("TENANT_GOVERNANCE_ENABLED") + else: + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", flag) + def forbidden(*args): + pytest.fail("disabled governance touched authority") + governance.register_governance_collaborators(app=app, tenant_resolver=forbidden) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not key_controls.model_allowed(USER, "other:gpt") + assert not budget_service.budgeted({"username": "alice"}) + + +def test_legacy_remains_unrestricted_by_ancestors(setup): + app, store = setup + policy(store, models=[]) + governance.register_governance_collaborators(app=app, store=store) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not governance.workspace_context(USER) + + +def test_grants_intersect_key_team_and_organisation_and_tools(setup): + app, store = setup + policy(store, models=["openai:*"], tools=["search"]) + policy(store, team="team1", models=["openai:gpt"], tools=[]) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not key_controls.model_allowed(USER, "openai:other") + assert not key_controls.model_allowed({**USER, "allowed_models": "other:*"}, "openai:gpt") + assert not governance.grant_allowed(True, "search", user=USER, kind="tools") + assert not governance.grant_allowed(False, "search", user=USER, kind="tools") + + +def test_atomic_reject_and_settlement_at_all_levels(setup): + app, store = setup + policy(store, daily=1_000_000, monthly=2_000_000) + policy(store, team="team1", daily=500_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert first.allowed + denied = budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW) + assert not denied.allowed and denied.error == "tenant_budget_exceeded" + assert denied.details["level"] == "team" + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + budget_service.BudgetService.complete(first.reservation, + {"selected_model": "openai:gpt", "cost_usd": 0.1, "cost_basis": "usage", "status": 200}) + assert budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW).allowed + + +def test_unknown_holds_survive_settle_and_disabled_flag(setup, monkeypatch): + app, store = setup + policy(store, daily=500_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + budget_service.BudgetService.complete(first.reservation, {"cost_usd": None, "cost_basis": None}) + budget_service.BudgetService.settle(first.reservation) + assert not budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW).allowed + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "false") + budget_service.BudgetService.settle(first.reservation) + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + + +def test_concurrent_race_has_one_winner_across_connections(setup): + app, store = setup + policy(store, daily=500_000) + def reserve(_): + with app.test_request_context(): + return budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(reserve, range(8))) + assert sum(result.allowed for result in results) == 1 + assert sum(result.error == "tenant_budget_exceeded" for result in results) == 7 + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + + +def test_missing_schema_fails_closed_before_key_reservation(setup): + app, _ = setup + empty = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(":memory:")) + governance.register_governance_collaborators(app=app, store=empty, tenant_resolver=lambda user: TENANT) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + key_controls.model_allowed(USER, "openai:gpt") + assert failure.value.code == "tenant_governance_unavailable" and failure.value.status == 503 + decision = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert decision.error == "tenant_governance_unavailable" and decision.status_code == 503 + assert budget_service.BudgetService._reservations == {} + + +def test_usage_isolation_attribution_and_role_permissions(setup): + app, store = setup + policy(store, daily=1_000_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW) + budget_service.BudgetService.complete(first.reservation, + {"selected_model": "openai:actual", "cost_usd": 0.1, "cost_basis": "usage"}) + second = budget_service.BudgetService.check_and_reserve({**USER, "username": "bob"}, 0.2, NOW) + budget_service.BudgetService.complete(second.reservation, + {"selected_model": "other:actual", "cost_usd": 0.15, "cost_basis": "usage"}) + own = store.call("usage", context=governance.context_dict(TENANT), role="member") + all_usage = store.call("usage", context=governance.context_dict(TENANT), role="billing") + foreign = store.call("usage", context=governance.context_dict(TenantContext("alice", "org2")), role="admin") + assert own["spent_micro_usd"] == 100_000 and all_usage["spent_micro_usd"] == 250_000 + assert foreign["spent_micro_usd"] == 0 + assert own["models"][0]["model"] == "openai:actual" and own["models"][0]["provider"] == "openai" + assert own["models"][0]["price_basis"] == "usage" + assert "organisation" not in own and "organisation" in all_usage + assert not governance.permitted("billing", "edit") and not governance.permitted("member", "budgets") + + +def test_registered_admin_routes_csrf_cas_and_no_role_writes(setup, monkeypatch): + app, store = setup + from routes import usage + monkeypatch.setattr(usage.AuthService, "get_current_user", lambda: {"username": "alice", "is_admin": True}) + csrf = CSRFProtect(app) + usage.register_usage_routes(app, csrf) + @app.route("/test-csrf") + def csrf_token(): + return {"token": generate_csrf()} + client = app.test_client() + with client.session_transaction() as session: + session["authenticated"] = True + session["user"] = "alice" + url = "/admin/organisations/org1/governance" + assert client.get(url).status_code == 200 + assert client.put(url, json={}).status_code == 400 + token = client.get("/test-csrf").json["token"] + headers = {"X-CSRFToken": token} + assert client.put(url, json={"daily": 1000}, headers=headers).status_code == 428 + headers["If-Match"] = '"0"' + updated = client.put(url, json={"daily": 1000}, headers=headers) + assert updated.status_code == 200 and updated.headers["ETag"] == '"1"' + assert client.put(url, json={"daily": 2000}, headers=headers).status_code == 412 + headers["If-Match"] = '"1"' + assert client.put(url, json={"role": "admin"}, headers=headers).status_code == 400 + assert client.get("/admin/organisations/foreign/governance").status_code == 403 + with store.connect() as db: + audit = db.execute("SELECT * FROM tenant_governance_audit").fetchall() + assert len(audit) == 1 + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "false") + assert client.get(url).status_code == 404 and client.put(url, json={}).status_code == 404 + + +def test_auto_eligibility_uses_same_intersection(setup, monkeypatch): + app, store = setup + from services import intelligence_policy + policy(store, models=[]) + monkeypatch.setattr(intelligence_policy, "get_adapter", lambda *args: object()) + candidate = {"model": "openai:gpt", "enabled": True, "entitled": True, "privacy_allowed": True, "billing": "free"} + with app.test_request_context(): + g.authenticated_user = USER + assert not intelligence_policy.eligible(candidate, False, {"API_BASE_URLS": {}}, model_status=lambda _: "enabled") + + +def test_missing_column_no_partial_state_and_repeatable_migration(setup): + app, store = setup + with store.connect() as db: + db.execute("CREATE TABLE old_rows (value TEXT)") + db.execute("INSERT INTO old_rows VALUES ('preserved')") + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + assert db.execute("SELECT value FROM old_rows").fetchone()[0] == "preserved" + db.execute("ALTER TABLE tenant_governance_policies RENAME COLUMN models TO old_models") + with app.test_request_context(): + result = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert result.status_code == 503 + with store.connect() as db: + assert db.execute("SELECT COUNT(*) FROM tenant_governance_reservations").fetchone()[0] == 0 + + +def test_key_reject_is_atomic_and_unknown_price_denied(setup): + app, store = setup + policy(store, daily=1_000_000) + with app.test_request_context(): + result = budget_service.BudgetService.check_and_reserve({**USER, "daily_budget_usd": 0.1}, 0.4, NOW) + assert result.error == "budget_exceeded" + assert budget_service.BudgetService.check_and_reserve(USER, None, NOW).error == "unpriced_reservation" + with store.connect() as db: + assert db.execute("SELECT COUNT(*) FROM tenant_governance_components").fetchone()[0] == 0 + assert db.execute("SELECT COUNT(*) FROM tenant_governance_baselines").fetchone()[0] == 0 + + +def test_resolver_cannot_change_principal_and_role_defaults_to_denied(setup): + app, store = setup + governance.register_governance_collaborators(app=app, store=store, tenant_resolver=lambda user: TenantContext("bob", "org1")) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.workspace_context(USER) + assert failure.value.status == 403 + governance.register_governance_collaborators(app=app, store=store, tenant_resolver=lambda user: TENANT) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.workspace_usage(USER) + assert failure.value.status == 403 + + +def test_invalid_flag_warns_once_without_value(monkeypatch, caplog): + governance._warn_once.cache_clear() + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "sensitive-invalid-value") + assert not governance.enabled() and not governance.enabled() + messages = [record.message for record in caplog.records if "TENANT_GOVERNANCE_ENABLED" in record.message] + assert len(messages) == 1 and "sensitive-invalid-value" not in messages[0] + + +def test_private_adapter_preserves_ancestor_denial_level_without_replaying(setup, monkeypatch): + app, _ = setup + calls = [] + monkeypatch.setattr(governance.control_state_d1, "using_d1", lambda: True) + def remote(endpoint, operation, **values): + calls.append((endpoint, operation, values)) + return {"version": 1, "decision": {"allowed": False, "code": "tenant_budget_exceeded", + "status": 429, "level": "team", "period": "daily"}} + monkeypatch.setattr(governance.control_state_d1, "call", remote) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.D1GovernanceStore().call("reserve", id="tg_test") + assert failure.value.code == "tenant_budget_exceeded" and failure.value.level == "team" + assert len(calls) == 1 and calls[0][0] == "tenant_governance" and calls[0][2]["rpc"] is True + + +def test_python_conflicting_settlement_cannot_replace_cost(setup): + app, _ = setup + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + row = {"selected_model": "openai:gpt", "cost_usd": 0.1, "cost_basis": "usage"} + budget_service.BudgetService.complete(first.reservation, row) + budget_service.BudgetService.complete(first.reservation, row) + with pytest.raises(governance.GovernanceError) as failure: + budget_service.BudgetService.complete(first.reservation, {**row, "cost_usd": 0.2}) + assert failure.value.status == 409 + + +def test_billing_can_read_but_not_write_policy(setup, monkeypatch): + app, store = setup + from routes import usage + monkeypatch.setattr(usage.AuthService, "get_current_user", lambda: {"username": "alice", "is_admin": True}) + governance.register_governance_collaborators(app=app, store=store, + tenant_resolver=lambda user: TENANT, membership_role=lambda principal, org: "billing") + app.config["WTF_CSRF_ENABLED"] = False + usage.register_usage_routes(app, CSRFProtect(app)) + client = app.test_client() + with client.session_transaction() as session: + session["authenticated"] = True + session["user"] = "alice" + url = "/admin/organisations/org1/teams/team1/governance" + assert client.get(url).status_code == 200 + assert client.put(url, json={"daily": 1}, headers={"If-Match": '"0"'}).status_code == 403 + + +class RegisteredGovernanceTest(UnifiedApiTestCase): + def setUp(self): + with patch("config.load_runtime_env", lambda: None): + super().setUp() + import os + os.environ.update({"TENANT_GOVERNANCE_ENABLED": "true", "USAGE_RESERVATIONS_ENABLED": "false", + "USAGE_LEDGER_ENABLED": "false", "MODEL_PRICING_USD_PER_MILLION": '{"opencode:*":{"input":1000,"output":1000}}'}) + self.path = Path(self.temp_dir.name) / "governance.sqlite3" + with sqlite3.connect(self.path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + self.store = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(self.path)) + governance.register_governance_collaborators(app=self.app, store=self.store, + tenant_resolver=lambda user: TenantContext(user["username"], "org1", "team1"), + membership_role=lambda principal, org: "member") + with patch.object(self.app_module.AuthService, "get_current_user", return_value={"username": "admin", "is_admin": True}): + self.key = self.app_module.AuthService.create_user("alice", scopes=["chat", "models"])["api_key"] + budget_service.BudgetService.reset() + + def tearDown(self): + budget_service.BudgetService.reset() + super().tearDown() + + def chat(self): + response = self._chat_response() + response._content = b'{"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":7}}' + with patch.object(self.app_module.ProxyService, "make_request", return_value=response) as upstream: + result = self.client.post("/v1/chat/completions", headers={"Authorization": f"Bearer {self.key}"}, + json={"model": "opencode:glm-5.2", "messages": [{"role": "user", "content": "test"}], "max_tokens": 5}, buffered=True) + return result, upstream + + def test_missing_table_refuses_real_route_before_upstream(self): + governance.register_governance_collaborators(app=self.app, + store=governance.SQLiteGovernanceStore(lambda: sqlite3.connect(":memory:")), + tenant_resolver=lambda user: TENANT) + response, upstream = self.chat() + assert response.status_code == 503 and response.json["error"]["code"] == "tenant_governance_unavailable" + upstream.assert_not_called() + + def test_workspace_budget_envelope_and_success_attribution(self): + policy(self.store, daily=1_000_000) + response, upstream = self.chat() + assert response.status_code == 200 + upstream.assert_called_once() + with self.store.connect() as db: + row = db.execute("SELECT provider,model,price_basis,charged FROM tenant_governance_reservations").fetchone() + assert row == ("opencode", "opencode:glm-5.2", "usage", 12000) + self.store.call("put", org_id="org1", team_id=None, actor="alice", revision=1, + policy={"daily": 12000}) + denied, upstream = self.chat() + assert denied.status_code == 429 and denied.json["error"]["code"] == "tenant_budget_exceeded" + assert denied.json["error"]["level"] == "organisation" + upstream.assert_not_called() + + def test_usage_route_never_reads_unscoped_history(self): + from routes import usage + with patch.object(usage, "usage_history", side_effect=AssertionError("unscoped history")): + result = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + assert result.status_code == 200 and result.json["workspace"]["org_id"] == "org1" + assert result.json["workspace"]["spent_micro_usd"] == 0 + + def test_disabled_and_legacy_usage_are_identical(self): + import os + os.environ["TENANT_GOVERNANCE_ENABLED"] = "false" + disabled = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + os.environ["TENANT_GOVERNANCE_ENABLED"] = "true" + governance.register_governance_collaborators(app=self.app, store=self.store) + legacy = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + assert disabled.status_code == legacy.status_code == 200 and disabled.data == legacy.data diff --git a/tests/test_tenant_governance_worker.mjs b/tests/test_tenant_governance_worker.mjs new file mode 100644 index 00000000..ef78afd5 --- /dev/null +++ b/tests/test_tenant_governance_worker.mjs @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { TenantContext } from "../worker/enterprise-contract.mjs"; +import { handleTenantGovernanceRequest, intersectGrants, createGovernanceLifecycle } from "../worker/tenant-governance-d1.mjs"; + +const NOW = Date.UTC(2026, 9, 9); +const context = { principal_id: "alice", org_id: "org1", team_id: "team1", grants_revision: 0 }; +const migration = readFileSync(new URL("../intelligence-migrations/0034_tenant_governance.sql", import.meta.url), "utf8"); +function database(t, schema = true) { + const sql = new DatabaseSync(":memory:"); + t.after(() => sql.close()); + if (schema) sql.exec(migration); + const db = { prepare(query) { + let values = []; + const statement = { bind(...args) { values = args; return statement; }, + all() { + const prepared = sql.prepare(query); + if (!prepared.columns().length) { + const result = prepared.run(...values); + return { success: true, results: [], meta: { changes: Number(result.changes) } }; + } + return { success: true, results: prepared.all(...values), meta: { changes: 0 } }; + }, first() { return sql.prepare(query).get(...values) ?? null; } }; + return statement; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { const results = statements.map(s => s.all()); sql.exec("COMMIT"); return results; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const env = { TENANT_GOVERNANCE_ENABLED: "true", INTELLIGENCE_DB: db }; + const call = async body => { + const response = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }), + }), env, { now: () => NOW }); + return { status: response.status, ...await response.json() }; + }; + return { sql, db, env, call }; +} +const put = (team_id = null, policy = {}) => ({ operation: "put", org_id: "org1", team_id, + actor: "alice", revision: 0, policy }); +const reserve = (id, changes = {}) => ({ operation: "reserve", id: `tg_${id}`, context, amount: 400_000, + key_daily: 1_000_000, key_monthly: null, base_day: 0, base_month: 0, + day: "2026-10-09", month: "2026-10", ...changes }); +const settle = (id, changes = {}) => ({ operation: "settle", id: `tg_${id}`, cost: 100_000, + provider: "openai", model: "openai:actual", price_basis: "usage", before_dispatch: false, ...changes }); + +test("off, empty and invalid configuration return 404 without storage", async () => { + for (const flag of [undefined, "", "false", "malformed"]) { + const response = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance"), + { TENANT_GOVERNANCE_ENABLED: flag, INTELLIGENCE_DB: { prepare() { assert.fail("disabled storage"); } } }); + assert.equal(response.status, 404); + } +}); + +test("missing schema, missing column and unavailable binding return JSON 503", async t => { + const { call, sql } = database(t, false); + assert.equal((await call(reserve("one"))).error.code, "tenant_governance_unavailable"); + sql.exec(migration); + sql.exec("ALTER TABLE tenant_governance_policies RENAME COLUMN models TO old_models"); + assert.equal((await call(reserve("two"))).status, 503); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_reservations").get().n, 0); +}); + +test("grant intersection never widens key or ancestor grants; absent rows allow", () => { + assert.equal(intersectGrants(true, "openai:gpt", [{ models: ["openai:*"] }, { models: ["openai:gpt"] }]), true); + assert.equal(intersectGrants(true, "openai:other", [{ models: ["openai:gpt"] }]), false); + assert.equal(intersectGrants(false, "openai:gpt", [{ models: ["*"] }]), false); + assert.equal(intersectGrants(true, "search", [{ tools: [] }], "tools"), false); + assert.equal(intersectGrants(true, "search", [] , "tools"), true); +}); + +test("policy CAS and content-free audit reject role escalation", async t => { + const { call, sql } = database(t); + assert.equal((await call(put(null, { daily: 500_000 }))).policy.revision, 1); + assert.equal((await call(put(null, { daily: 600_000 }))).status, 412); + assert.equal((await call({ ...put(null, { role: "admin" }), revision: 1 })).status, 400); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_audit").get().n, 1); + assert.equal((await call({ operation: "policies", context })).policies.length, 1); +}); + +test("every level rejects atomically, settles together and holds unknown costs", async t => { + const { call, sql } = database(t); + await call(put(null, { daily: 1_000_000 })); + await call(put("team1", { daily: 500_000 })); + assert.equal((await call(reserve("one"))).status, 200); + const denied = await call(reserve("two", { amount: 200_000 })); + assert.equal(denied.status, 429); assert.equal(denied.error.level, "team"); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 6); + assert.equal((await call(settle("one"))).status, 200); + assert.equal((await call(reserve("three"))).status, 200); + await call(settle("three", { cost: null, price_basis: null })); + assert.equal((await call(reserve("four", { amount: 1 }))).status, 429); + const usage = await call({ operation: "usage", context, role: "member" }); + assert.equal(usage.spent_micro_usd, 100_000); assert.equal(usage.holds_micro_usd, 400_000); + assert.equal(usage.models.find(row => row.price_basis === "usage").model, "openai:actual"); +}); + +test("concurrent request race admits exactly one; rejection writes nothing", async t => { + const { call, sql } = database(t); + await call(put(null, { monthly: 500_000 })); + const results = await Promise.all(Array.from({ length: 12 }, (_, n) => call(reserve(String(n))))); + assert.equal(results.filter(result => result.status === 200).length, 1); + assert.equal(results.filter(result => result.status === 429).length, 11); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_reservations").get().n, 1); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 2); +}); + +test("key exhaustion denies without touching ancestor or baseline state", async t => { + const { call, sql } = database(t); + const result = await call(reserve("one", { base_day: 800_000 })); + assert.equal(result.status, 429); assert.equal(result.error.code, "budget_exceeded"); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 0); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 0); +}); + +test("private RPC preserves ancestor level and missing-schema remains HTTP 503", async t => { + const { call } = database(t); + await call(put(null, { daily: 100_000 })); + const denied = await call({ ...reserve("one"), rpc: true }); + assert.equal(denied.status, 200); + assert.equal(denied.decision.code, "tenant_budget_exceeded"); + assert.equal(denied.decision.level, "organisation"); + const absent = database(t, false); + assert.equal((await absent.call({ ...reserve("two"), rpc: true })).status, 503); +}); + +test("usage is scoped to caller workspace and own principal unless billing/admin", async t => { + const { call } = database(t); + await call(reserve("one")); await call(settle("one")); + await call(reserve("two", { context: { ...context, principal_id: "bob" } })); await call(settle("two")); + await call(reserve("three", { context: { ...context, org_id: "org2" } })); await call(settle("three")); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + const billing = await call({ operation: "usage", context, role: "billing" }); + assert.equal(billing.spent_micro_usd, 200_000); assert.equal(billing.organisation.spent_micro_usd, 200_000); + assert.equal((await call({ operation: "usage", context, role: "stranger" })).status, 403); +}); + +test("private transport rejects public URLs, content, oversized bodies and invalid money", async t => { + const { call, env } = database(t); + assert.equal((await call({ ...reserve("one"), prompt: "content" })).status, 400); + for (const amount of [-1, 1.5, Number.MAX_SAFE_INTEGER, null]) { + assert.equal((await call(reserve("one", { amount }))).status, 400); + } + const response = await handleTenantGovernanceRequest(new Request("https://public.example/v1/tenant-governance", { method: "POST" }), env); + assert.equal(response.status, 404); + const huge = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ operation: "put", junk: "x".repeat(33000) }), + }), env); + assert.equal(huge.status, 400); +}); + +test("migration is additive and repeatable with existing rows", t => { + const { sql } = database(t); + sql.exec("CREATE TABLE old_usage (id TEXT); INSERT INTO old_usage VALUES ('kept');"); + sql.exec(migration); + assert.equal(sql.prepare("SELECT id FROM old_usage").get().id, "kept"); +}); + +test("duplicate reservation IDs cannot add components in another scope", async t => { + const { call, sql } = database(t); + assert.equal((await call(reserve("same"))).status, 200); + assert.equal((await call(reserve("same", { context: { ...context, org_id: "org2", principal_id: "bob" } }))).status, 409); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 6); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 2); +}); + +test("settlement is idempotent; conflicting amounts cannot replace settled cost", async t => { + const { call } = database(t); + await call(reserve("one")); await call(settle("one")); + assert.equal((await call(settle("one"))).status, 200); + assert.equal((await call(settle("one", { cost: 200_000 }))).status, 409); +}); + +test("native lifecycle uses injected verified tenant, intersection and atomic quotas", async t => { + const { env, call } = database(t); + await call(put(null, { daily: 500_000, models: ["openai:*"] })); + const hooks = createGovernanceLifecycle(env, { call: async body => call(body), + tenant_resolver: async () => new TenantContext(context) }); + await hooks.admit({ user: { id: "alice" }, model: "openai:gpt", key_allowed: true, amount: 400_000, + key_daily: 1_000_000, base_day: 0, base_month: 0 }); + await hooks.before_dispatch(); + await hooks.finalize({ usage: { selected_model: "openai:actual", cost_micro_usd: 100_000, cost_basis: "usage" } }); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + const denied = createGovernanceLifecycle(env, { call: async body => call(body), tenant_resolver: async () => new TenantContext(context) }); + await assert.rejects(denied.admit({ model: "other:gpt", key_allowed: true }), /model_not_allowed/); +}); + +test("disabled and legacy native lifecycle do not call storage", async () => { + for (const flag of ["", "true"]) { + const hooks = createGovernanceLifecycle({ TENANT_GOVERNANCE_ENABLED: flag }, { + call: () => assert.fail("legacy storage"), tenant_resolver: async () => new TenantContext({ principal_id: "alice" }), + }); + await hooks.admit({}); await hooks.before_dispatch(); await hooks.finalize({}); + } +}); + +test("native flat cache events release holds and ancestor errors retain the level", async t => { + const { env, call } = database(t); + await call(put(null, { daily: 500_000 })); + const create = () => createGovernanceLifecycle(env, { call, tenant_resolver: async () => new TenantContext(context) }); + const cached = create(); + await cached.admit({ model: "openai:gpt", key_allowed: true, amount: 400_000 }); + await cached.finalize({ cost_basis: "cache", selected_model: "openai:gpt", handedOff: false }); + assert.equal((await call({ operation: "usage", context, role: "member" })).holds_micro_usd, 0); + const held = create(); + await held.admit({ model: "openai:gpt", key_allowed: true, amount: 400_000 }); + await held.before_dispatch(); + await held.finalize({ cost_basis: "usage", cost_micro_usd: 100_000, selected_model: "openai:actual", handedOff: true }); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + await assert.rejects(create().admit({ model: "openai:gpt", key_allowed: true, amount: 500_000 }), + error => error.status === 429 && error.details.level === "organisation"); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..16038c05 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,7 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "TENANT_GOVERNANCE_ENABLED", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..8a1ab79e 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "tenant_governance_policies", "tenant_governance_reservations", "tenant_governance_components", "tenant_governance_baselines", "tenant_governance_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/tenant-governance-d1.mjs b/worker/tenant-governance-d1.mjs new file mode 100644 index 00000000..aef3f34f --- /dev/null +++ b/worker/tenant-governance-d1.mjs @@ -0,0 +1,315 @@ +/** Private governance domain: intersected grants, CAS policy and atomic quota components. */ +import { TenantContext, AuthorityOperation, legacyTenant, MAX_INTEGER } from "./enterprise-contract.mjs"; + +const PERMISSIONS = Object.freeze({ admin: ["edit", "budgets", "usage", "all_usage"], + billing: ["budgets", "usage", "all_usage"], member: ["usage"] }); +const FIELDS = { + get: ["org_id", "team_id"], put: ["org_id", "team_id", "actor", "revision", "policy"], + policies: ["context"], usage: ["context", "role", "since", "until"], + reserve: ["id", "context", "amount", "day", "month", "key_daily", "key_monthly", "base_day", "base_month"], + dispatch: ["id"], settle: ["id", "cost", "provider", "model", "price_basis", "before_dispatch"], +}; +const SCHEMA = [ + "SELECT org_id,team_id,revision,models,tools,daily,monthly FROM tenant_governance_policies LIMIT 0", + "SELECT id,principal_id,org_id,team_id,day,month,estimate,charged,state,revision,provider,model,price_basis,created_at,operation_id FROM tenant_governance_reservations LIMIT 0", + "SELECT reservation_id,level,scope_org,scope_id,period_kind,period,limit_units FROM tenant_governance_components LIMIT 0", + "SELECT principal_id,period,amount FROM tenant_governance_baselines LIMIT 0", + "SELECT operation_id,actor,org_id,team_id,revision,kind,at FROM tenant_governance_audit LIMIT 0", +]; +let warned = false; +export class GovernanceError extends Error { + constructor(code = "tenant_governance_unavailable", status = 503, details = {}) { + super(code); Object.assign(this, { code, status, details }); + } +} +const fail = (code = "invalid_governance_operation", status = 400, details = {}) => { throw new GovernanceError(code, status, details); }; +export function governanceEnabled(env = {}, warn = message => console.warn(message)) { + const raw = env.TENANT_GOVERNANCE_ENABLED; + const flag = raw === undefined ? "" : typeof raw === "string" ? raw.trim().toLowerCase() : "invalid"; + if (["", "false", "0", "off", "no"].includes(flag)) return false; + if (["true", "1", "on", "yes"].includes(flag)) return true; + if (!warned) { warned = true; warn("Invalid TENANT_GOVERNANCE_ENABLED; tenant governance disabled"); } + return false; +} +const permitted = (role, permission) => Object.hasOwn(PERMISSIONS, role) && PERMISSIONS[role].includes(permission); +const integer = n => Number.isSafeInteger(n) && n >= 0 && n < MAX_INTEGER; +function opaque(value) { + if (typeof value !== "string" || !/^[A-Za-z0-9_:.\-]{1,128}$/.test(value)) fail(); +} +function policy(value) { + if (!value || typeof value !== "object" || Array.isArray(value) || + Object.keys(value).some(key => !["models", "tools", "daily", "monthly"].includes(key))) fail(); + const result = { models: value.models ?? null, tools: value.tools ?? null, + daily: value.daily ?? null, monthly: value.monthly ?? null }; + for (const name of ["models", "tools"]) { + if (result[name] !== null && (!Array.isArray(result[name]) || result[name].length > 64 || + result[name].some(pattern => typeof pattern !== "string" || !/^[A-Za-z0-9._:/+@*\-]{1,256}$/.test(pattern)))) fail(); + } + for (const name of ["daily", "monthly"]) if (result[name] !== null && !integer(result[name])) fail(); + return result; +} +function decoded(row) { + try { + return { revision: row.revision, ...policy({ models: row.models === null ? null : JSON.parse(row.models), + tools: row.tools === null ? null : JSON.parse(row.tools), daily: row.daily, monthly: row.monthly }) }; + } catch { throw new GovernanceError(); } +} +export function intersectGrants(keyAllowed, candidate, policies, kind = "models") { + if (!["models", "tools"].includes(kind)) fail(); + return keyAllowed && policies.every(row => row[kind] == null || typeof candidate === "string" && + row[kind].some(pattern => new RegExp(`^${pattern.toLowerCase().split("*") + .map(part => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`).test(candidate.trim().toLowerCase()))); +} +function validate(body) { + if (!body || typeof body !== "object" || Array.isArray(body) || body.version !== 1 || !Object.hasOwn(FIELDS, body.operation)) fail(); + const allowed = new Set(["version", "operation", "rpc", ...FIELDS[body.operation]]); + if (Object.keys(body).some(key => !allowed.has(key))) fail(); + if (body.rpc !== undefined && typeof body.rpc !== "boolean") fail(); + const op = body.operation; + if (["get", "put"].includes(op)) { + opaque(body.org_id); if (body.team_id != null) opaque(body.team_id); + if (op === "put") { opaque(body.actor); if (!integer(body.revision)) fail(); body.policy = policy(body.policy); } + } + if (["policies", "usage", "reserve"].includes(op)) { + try { body.context = new TenantContext(body.context); } catch { fail(); } + if (body.context.org_id === null) fail("tenant_scope_denied", 403); + } + if (op === "usage" && !permitted(body.role, "usage")) fail("tenant_role_denied", 403); + if (op === "usage") for (const key of ["since", "until"]) { + if (body[key] !== undefined && (typeof body[key] !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(body[key]))) fail(); + } + if (["reserve", "dispatch", "settle"].includes(op)) opaque(body.id); + if (op === "reserve") { + for (const key of ["amount", "base_day", "base_month"]) if (!integer(body[key])) fail(); + for (const key of ["key_daily", "key_monthly"]) if (body[key] !== null && !integer(body[key])) fail(); + if (typeof body.day !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(body.day) || + typeof body.month !== "string" || body.month !== body.day.slice(0, 7)) fail(); + } + if (op === "settle") { + if (body.cost !== null && !integer(body.cost)) fail(); + if (body.before_dispatch !== undefined && typeof body.before_dispatch !== "boolean") fail(); + if (body.price_basis != null && !["usage", "estimate", "cache", "released"].includes(body.price_basis)) fail(); + if (body.cost !== null && !["usage", "cache", "released"].includes(body.price_basis)) fail(); + if (body.before_dispatch && (body.cost !== 0 || body.price_basis !== "released")) fail(); + for (const key of ["provider", "model"]) if (body[key] != null && + (typeof body[key] !== "string" || !/^[A-Za-z0-9._:/+@\-]{1,256}$/.test(body[key]))) fail(); + } + return body; +} +async function batch(db, statements) { + const results = await db.batch(statements); + if (!Array.isArray(results) || results.some(result => result.success === false)) throw new GovernanceError(); + return results; +} +async function policies(db, context) { + const rows = await db.prepare("SELECT * FROM tenant_governance_policies WHERE org_id=? AND (team_id='' OR team_id=?)") + .bind(context.org_id, context.team_id ?? "").all(); + return { policies: rows.results.map(decoded) }; +} +async function admin(db, body, now) { + const org = body.org_id, team = body.team_id ?? ""; + if (body.operation === "get") { + const row = await db.prepare("SELECT * FROM tenant_governance_policies WHERE org_id=? AND team_id=?").bind(org, team).first(); + return { policy: row ? decoded(row) : { revision: 0, models: null, tools: null, daily: null, monthly: null } }; + } + const operationId = crypto.randomUUID(); + const { models, tools, daily, monthly } = body.policy; + const results = await batch(db, [ + db.prepare(`INSERT INTO tenant_governance_audit + SELECT ?,?,?,?,?+1,'policy',? WHERE COALESCE((SELECT revision FROM tenant_governance_policies + WHERE org_id=? AND team_id=?),0)=?`).bind(operationId, body.actor, org, team, body.revision, + new Date(now).toISOString(), org, team, body.revision), + db.prepare(`INSERT INTO tenant_governance_policies (org_id,team_id,revision,models,tools,daily,monthly) + SELECT ?,?,?,?,?,?,? WHERE EXISTS (SELECT 1 FROM tenant_governance_audit WHERE operation_id=?) + ON CONFLICT(org_id,team_id) DO UPDATE SET revision=excluded.revision,models=excluded.models, + tools=excluded.tools,daily=excluded.daily,monthly=excluded.monthly`) + .bind(org, team, body.revision + 1, models === null ? null : JSON.stringify(models), + tools === null ? null : JSON.stringify(tools), daily, monthly, operationId), + db.prepare("SELECT p.* FROM tenant_governance_policies p WHERE org_id=? AND team_id=? AND EXISTS (SELECT 1 FROM tenant_governance_audit WHERE operation_id=?)") + .bind(org, team, operationId), + ]); + if (!results[2].results.length) fail("revision_stale", 412); + return { policy: decoded(results[2].results[0]) }; +} + +// All scope reads, admission and component writes run in one D1 batch transaction. +const SCOPES = `WITH scopes(level,scope_org,scope_id,period_kind,period,limit_units) AS ( + SELECT 'key','',?1,'daily',?4,?6 UNION ALL SELECT 'key','',?1,'monthly',?5,?7 + UNION ALL SELECT 'organisation',?2,?2,'daily',?4,(SELECT daily FROM tenant_governance_policies WHERE org_id=?2 AND team_id='') + UNION ALL SELECT 'organisation',?2,?2,'monthly',?5,(SELECT monthly FROM tenant_governance_policies WHERE org_id=?2 AND team_id='') + UNION ALL SELECT 'team',?2,?3,'daily',?4,(SELECT daily FROM tenant_governance_policies WHERE org_id=?2 AND team_id=?3) WHERE ?3!='' + UNION ALL SELECT 'team',?2,?3,'monthly',?5,(SELECT monthly FROM tenant_governance_policies WHERE org_id=?2 AND team_id=?3) WHERE ?3!='' +), totals AS (SELECT s.*, COALESCE((SELECT SUM(CASE WHEN r.state!='settled' THEN r.estimate + WHEN c.period=s.period THEN r.charged ELSE 0 END) FROM tenant_governance_components c + JOIN tenant_governance_reservations r ON r.id=c.reservation_id WHERE c.level=s.level AND c.scope_org=s.scope_org + AND c.scope_id=s.scope_id AND c.period_kind=s.period_kind),0) + + CASE WHEN s.level='key' THEN COALESCE((SELECT amount FROM tenant_governance_baselines WHERE principal_id=?1 AND period=s.period), + CASE WHEN s.period_kind='daily' THEN ?10 ELSE ?11 END) ELSE 0 END AS used FROM scopes s +), denied AS (SELECT * FROM totals WHERE limit_units IS NOT NULL AND (used>=limit_units OR ?8>limit_units-used))`; + +async function reserve(db, body, now) { + const c = body.context; + const args = [c.principal_id, c.org_id, c.team_id ?? "", body.day, body.month, body.key_daily, body.key_monthly, + body.amount, body.id, body.base_day, body.base_month, new Date(now).toISOString(), crypto.randomUUID()]; + const admission = `${SCOPES} INSERT INTO tenant_governance_reservations + (id,principal_id,org_id,team_id,day,month,estimate,state,created_at,operation_id) + SELECT ?9,?1,?2,?3,?4,?5,?8,'reserved',?12,?13 WHERE NOT EXISTS (SELECT 1 FROM denied) + ON CONFLICT(id) DO NOTHING RETURNING id`; + const results = await batch(db, [ + db.prepare(admission).bind(...args), + db.prepare(`${SCOPES} INSERT OR IGNORE INTO tenant_governance_components + SELECT ?9,level,scope_org,scope_id,period_kind,period,limit_units FROM scopes + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?9 AND operation_id=?13)`) + .bind(...args), + db.prepare(`INSERT OR IGNORE INTO tenant_governance_baselines SELECT ?1,?2,?3 + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?4 AND operation_id=?5)`) + .bind(c.principal_id, body.day, body.base_day, body.id, args[12]), + db.prepare(`INSERT OR IGNORE INTO tenant_governance_baselines SELECT ?1,?2,?3 + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?4 AND operation_id=?5)`) + .bind(c.principal_id, body.month, body.base_month, body.id, args[12]), + db.prepare(`${SCOPES} SELECT level,period_kind FROM denied ORDER BY CASE level WHEN 'key' THEN 0 WHEN 'organisation' THEN 1 ELSE 2 END LIMIT 1`) + .bind(...args.slice(0, 11)), + db.prepare("SELECT id FROM tenant_governance_reservations WHERE id=?").bind(body.id), + ]); + if (!results[0].results.length) { + if (results[5].results.length) fail("reservation_conflict", 409); + const denied = results[4].results[0]; + if (!denied) fail("reservation_conflict", 409); + fail(denied.level === "key" ? "budget_exceeded" : "tenant_budget_exceeded", 429, + { level: denied.level, period: denied.period_kind }); + } + return { id: body.id }; +} +async function settle(db, body) { + const state = body.operation === "dispatch" ? "dispatched" : body.cost === null ? "unknown" : "settled"; + const row = await db.prepare("SELECT * FROM tenant_governance_reservations WHERE id=?").bind(body.id).first(); + if (!row) fail("reservation_conflict", 409); + if (["settled", "unknown"].includes(row.state)) { + if (row.state === "settled" && body.operation === "settle" && body.cost !== null && row.charged !== body.cost) fail("reservation_conflict", 409); + return { state: row.state }; + } + if (body.before_dispatch && row.state !== "reserved") fail("reservation_conflict", 409); + const results = await batch(db, [db.prepare(`UPDATE tenant_governance_reservations SET state=?,charged=?, + provider=COALESCE(?,provider),model=COALESCE(?,model),price_basis=COALESCE(?,price_basis),revision=revision+1 + WHERE id=? AND revision=? AND state IN ('reserved','dispatched') RETURNING state`) + .bind(state, body.cost ?? null, body.provider ?? null, body.model ?? null, body.price_basis ?? null, body.id, row.revision)]); + if (!results[0].results.length) fail("reservation_conflict", 409); + return { state }; +} +async function usage(db, body) { + const c = body.context, all = permitted(body.role, "all_usage"); + const where = "org_id=? AND (?='' OR team_id=?) AND (?=1 OR principal_id=?) AND day>=? AND day<=?"; + const since = body.since ?? "0000-01-01", until = body.until ?? "9999-12-31"; + const args = [c.org_id, c.team_id ?? "", c.team_id ?? "", all ? 1 : 0, c.principal_id, since, until]; + const results = await batch(db, [ + db.prepare(`SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd, + COUNT(*) AS requests FROM tenant_governance_reservations WHERE ${where}`).bind(...args), + db.prepare(`SELECT provider,model,price_basis,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE ${where} GROUP BY provider,model,price_basis ORDER BY model LIMIT 200`).bind(...args), + ...(all ? [db.prepare(`SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND day>=? AND day<=?`).bind(c.org_id, since, until)] : []), + db.prepare(`SELECT day,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE ${where} GROUP BY day ORDER BY day LIMIT 90`).bind(...args), + ]); + const totals = results[0].results[0]; + return { org_id: c.org_id, team_id: c.team_id, principal_id: c.principal_id, + cost_description: "Gateway cost estimates, not invoices", ...totals, models: results[1].results, + daily: results[all ? 3 : 2].results, range: { since, until }, + ...(all ? { organisation: results[2].results[0], team: c.team_id ? totals : null } : {}) }; +} +async function readBody(request) { + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json" || !request.body) fail(); + const reader = request.body.getReader(), chunks = []; + let length = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + length += value.byteLength; + if (length > 32768) fail(); + chunks.push(value); + } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + return validate(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + } catch (error) { if (error instanceof GovernanceError) throw error; fail(); } + finally { void reader.cancel().catch(() => {}); reader.releaseLock(); } +} +const reply = (value, status = 200) => Response.json({ version: 1, ...value }, + { status, headers: { "cache-control": "no-store", "x-content-type-options": "nosniff" } }); + +/** Mount only in the authenticated private managed-state dispatcher. */ +export async function handleTenantGovernanceRequest(request, env, { now = Date.now } = {}) { + if (!governanceEnabled(env)) return reply({ error: { code: "not_found" } }, 404); + let rpc = false; + try { + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/tenant-governance" || url.search || url.hash) fail("not_found", 404); + if (request.method !== "POST") fail("method_not_allowed", 405); + const body = await readBody(request), db = env.INTELLIGENCE_DB; + rpc = body.rpc === true; + if (!db?.prepare || !db?.batch) throw new GovernanceError(); + for (const query of SCHEMA) await db.prepare(query).all(); + let result; + if (["get", "put"].includes(body.operation)) result = await admin(db, body, now()); + else if (body.operation === "policies") result = await policies(db, body.context); + else if (body.operation === "reserve") result = await reserve(db, body, now()); + else if (body.operation === "usage") result = await usage(db, body); + else result = await settle(db, body); + return reply(result); + } catch (error) { + const failure = error instanceof GovernanceError ? error : new GovernanceError(); + // The bounded Container transport keeps only the code of HTTP errors. + if (rpc && failure.code === "tenant_budget_exceeded") { + return reply({ decision: { allowed: false, code: failure.code, status: failure.status, ...failure.details } }); + } + return reply({ error: { code: failure.code, ...failure.details } }, failure.status); + } +} + +function defaultTenantResolver(event) { + const context = new TenantContext({ principal_id: event.user?.id ?? event.user?.username ?? "legacy" }); + return legacyTenant(new AuthorityOperation({ context, scoped_id: context.principal_id, revision: 0, operation_id: "governance.resolve" })); +} +/** Named native hooks consume trusted admission metadata; no caller header selects tenancy. */ +export function createGovernanceLifecycle(env, { call, tenant_resolver = defaultTenantResolver }) { + let id, finished = false; + const rpc = async (operation, values) => { + const result = await call({ version: 1, operation, ...values }); + const failure = result.error ?? (result.decision?.allowed === false ? result.decision : null); + if (failure) throw new GovernanceError(failure.code, failure.status ?? result.status ?? 503, + { ...(failure.level ? { level: failure.level } : {}), ...(failure.period ? { period: failure.period } : {}) }); + return result; + }; + return { + async admit(event) { + if (!governanceEnabled(env)) return; + const context = await tenant_resolver(event); + if (!(context instanceof TenantContext)) fail("tenant_scope_denied", 403); + if (context.org_id === null) return; + const rows = (await rpc("policies", { context })).policies; + if (!intersectGrants(event.key_allowed === true, event.model, rows)) fail("model_not_allowed", 403); + for (const tool of event.tools ?? []) if (!intersectGrants(event.key_tools?.includes(tool) === true, tool, rows, "tools")) fail("tool_not_allowed", 403); + const candidateId = `tg_${crypto.randomUUID().replaceAll("-", "")}`; + const day = new Date().toISOString().slice(0, 10); + await rpc("reserve", { id: candidateId, context, amount: event.amount, day, month: day.slice(0, 7), + key_daily: event.key_daily ?? null, key_monthly: event.key_monthly ?? null, + base_day: event.base_day ?? 0, base_month: event.base_month ?? 0 }); + id = candidateId; + }, + async before_dispatch() { if (id && !finished) await rpc("dispatch", { id }); }, + async finalize(event) { + if (!id || finished) return; + const row = event.usage ?? event, model = row.selected_model ?? null; + const cached = row.cost_basis === "cache"; + const released = event.handedOff === false; + const measured = !event.cancellationOutcome?.ambiguous && row.cost_basis === "usage"; + await rpc("settle", { id, cost: released || cached ? 0 : measured ? row.cost_micro_usd ?? null : null, + provider: model?.includes(":") ? model.split(":", 1)[0] : null, model, + price_basis: cached ? "cache" : released ? "released" : row.cost_basis ?? null, before_dispatch: released && !cached }); + finished = true; + }, + }; +} From 048e2ffcd6071ad3b402c22217c24c209f65c043 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:37:33 +0530 Subject: [PATCH 6/9] feat(tenants): add opt-in organisations, teams and verified workspaces ORGANISATIONS_ENABLED (off by default) adds a two-level organisation and team hierarchy with stable opaque IDs, roles, caps and deactivation instead of deletion. Administrators manage it under /admin/organisations with If-Match revisions and audit rows. Each account has at most one server-side workspace binding; /v1/workspaces lists the caller's memberships and /v1/workspaces/switch changes the binding only after a membership check. A new first post-authentication hook resolves the verified TenantContext and refuses a missing or deactivated workspace with 404 or 403 before any storage, cache, claim or provider call. Caller headers never choose a workspace. Native Worker generations, knowledge and Realtime resolve the same context, only after the key matches. Accounts without a binding stay in the legacy namespace (migration 0033). --- app.py | 2 + docs/tenant-hierarchy.md | 109 ++++++ .../0033_tenant_hierarchy.sql | 30 ++ package.json | 2 +- routes/tenants.py | 127 ++++++ services/auth_service.py | 4 + services/gateway_extensions.py | 19 +- services/tenant_hierarchy.py | 366 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 11 + tests/test_tenant_hierarchy.py | 353 +++++++++++++++++ tests/test_tenant_hierarchy_worker.mjs | 341 ++++++++++++++++ worker/container-env.mjs | 1 + worker/d1-schema.mjs | 2 +- worker/gateway-extensions.mjs | 12 + worker/knowledge-edge.mjs | 20 +- worker/managed-state-dispatch.mjs | 2 + worker/realtime.mjs | 16 +- worker/tenants-d1.mjs | 262 +++++++++++++ 18 files changed, 1671 insertions(+), 8 deletions(-) create mode 100644 docs/tenant-hierarchy.md create mode 100644 intelligence-migrations/0033_tenant_hierarchy.sql create mode 100644 routes/tenants.py create mode 100644 services/tenant_hierarchy.py create mode 100644 tests/test_tenant_hierarchy.py create mode 100644 tests/test_tenant_hierarchy_worker.mjs create mode 100644 worker/tenants-d1.mjs diff --git a/app.py b/app.py index b6b09df7..b74df539 100644 --- a/app.py +++ b/app.py @@ -20,6 +20,7 @@ ) from routes.config_snapshots import register_config_snapshot_routes from routes.enterprise_preview import register_enterprise_preview_routes +from routes.tenants import register_tenant_routes from routes.core import register_core_routes from routes.dashboard_security import register_dashboard_security_routes from routes.intelligence_media import register_intelligence_media_routes @@ -147,6 +148,7 @@ def create_app() -> Flask: register_workbench_routes(app) register_config_snapshot_routes(app) register_enterprise_preview_routes(app) + register_tenant_routes(app, csrf=csrf) register_shadow_eval_routes(app, csrf, AuthService, MetricsService, ProxyService) register_knowledge_routes(app, csrf) register_gateway_mcp_routes(app, csrf) diff --git a/docs/tenant-hierarchy.md b/docs/tenant-hierarchy.md new file mode 100644 index 00000000..e6bff089 --- /dev/null +++ b/docs/tenant-hierarchy.md @@ -0,0 +1,109 @@ +# Organisations and teams + +`ORGANISATIONS_ENABLED` is off by default. Empty values mean off. Malformed values +mean off and produce one warning without the value. With the feature off, the +organisation and workspace endpoints return JSON 404 with `Cache-Control: no-store` +before authentication, CSRF checks or JSON parsing. Existing accounts, key records, +provider traffic and storage formats remain unchanged. Every principal has the +legacy tenant context and an empty namespace. + +Enable with `ORGANISATIONS_ENABLED=true` only after applying +`intelligence-migrations/0033_tenant_hierarchy.sql` to the configured account SQLite +database, or to D1 when `INTELLIGENCE_STORAGE_BACKEND=d1`. Missing tables or columns +return JSON 503 `tenant_storage_unavailable`, including for accounts without a +binding. There is no storage fallback or automatic account migration. + +The hierarchy has exactly two levels: organisation and team. IDs are generated, +opaque and immutable. Names may have up to 200 characters. Status is `active` or +`deactivated`. Each organisation retains at most 100 teams and 1,000 memberships, +including deactivated rows. The limit returns 409 `tenant_limit_reached`. +Membership roles are `admin`, `billing` and `member`; these labels do not grant +access to the gateway administrator dashboard. Each membership links an existing +account to one organisation and optionally one team in that organisation. + +## Administration + +The existing administrator dashboard session and CSRF token protect these routes: + +- `GET /admin/organisations` lists organisations; `POST` creates one with + `{"name":"Example"}` and returns 201. +- `GET /admin/organisations/` reads one; `PATCH` changes its name or status. +- `GET /admin/organisations//teams` lists teams; `POST` creates one with a name. +- `PATCH /admin/organisations//teams/` changes a team's name or status. +- `GET /admin/organisations//members` lists memberships. +- `PUT /admin/organisations//members/` sets role, team and status. + +All updates require `If-Match: ""`. A new membership requires `"0"`; +existing resources start at revision 1. Missing revisions return 428 +`revision_required`; malformed revisions return 400 `invalid_revision`; stale +revisions return 412 `revision_conflict`. Resource and binding responses expose +`ETag: ""`. Each successful update increments the revision. Team creation +does not change its parent organisation's revision. + +For example, to grant a team membership and explicitly select it for that account, +PUT a membership with `If-Match: "0"` and this body, using IDs returned by creation: + +```json +{"role":"member","team_id":"","status":"active","bind":true,"binding_revision":0} +``` + +`binding_revision` is the account's current binding revision (zero when unbound). +The membership, binding and audit rows commit together. A stale binding revision +leaves both old records intact. Set `status` to `deactivated` to revoke access; +rows and IDs remain stored. There is no deletion endpoint. Every successful +mutation records actor, action, target IDs, old/new revisions and UTC time. Audit +records exclude names, request bodies, keys and provider content. + +## Workspaces + +`GET /v1/workspaces` uses a gateway key with the `models` scope and returns only +its active memberships plus its current binding. Administrators also hold this +scope. Unbound accounts have binding revision zero and remain in the legacy +namespace even when they have memberships. + +`POST /v1/workspaces/switch` uses the same key and scope, requires +`If-Match: ""`, and accepts exactly +`{"org_id":"","team_id":""}`. Use `null` for an organisation-only +membership. The caller can select only its own active membership and its exact +team. The stored binding applies to the next request; existing request contexts +are unchanged. Dashboard CSRF applies to admin mutations; gateway-key workspace +selection uses the existing API CSRF exemption. + +Knowledge and Realtime resolve the workspace only after verifying the credential +for one principal. Accounts sharing a key prefix do not resolve each other's +workspaces. Their authenticated principals carry the validated `tenant_context`; +with organisations off, resolution returns the legacy context without querying +tenant storage. + +Resolution uses server storage and ignores `X-Org`, `X-Team`, +`X-MultiLLM-Workspace` and tenant fields in provider request bodies. A missing or +foreign organisation/team is 404 `workspace_not_found`; a deactivated +organisation/team, revoked membership or mismatched membership team is 403 +`workspace_forbidden`. A rejected binding prevents provider dispatch and later +policy, cache, claim and admission hooks. An administrator can repair a revoked +binding through the members endpoint. No automatic reparenting takes place. + +Python consumers use `current_tenant()` and `tenant_namespace()` from +`services.tenant_hierarchy`; native consumers use the validated `TenantContext` +from the tenant resolver. Namespaces are `org:` or `org:/team:`; the +legacy namespace is the empty string. Resolution rereads authoritative revisions +every request and has no process-local tenant cache. + +## Operational limits + +The private D1 domain is POST-only at +`http://intelligence.internal/v1/managed-state/tenants`, with a maximum request +size of 8,192 bytes. Writes use conditional SQL and an atomic batch with their +audit rows. Private Flask calls have bounded concurrency and wall time and never +replay an uncertain write. Standalone SQLite uses the configured account database +and serializes writes with an immediate transaction. + +Workspace identity does not introduce quotas, credits, payment collection, cost +allocation or provider discounts. Existing cache, idempotency, hosted-state and +usage ownership formats remain unchanged; a tenant binding alone does not isolate +those records. Consumers must incorporate the verified namespace before using +organisation isolation for shared records. Existing record ownership never moves +because an account selects a workspace. Audit and hierarchy rows are retained +indefinitely; content-retention policies do not erase them. Deactivation keeps +rows and continues to count toward the caps. There are no live-provider capability +or metering guarantees from these APIs. diff --git a/intelligence-migrations/0033_tenant_hierarchy.sql b/intelligence-migrations/0033_tenant_hierarchy.sql new file mode 100644 index 00000000..0b4dc73e --- /dev/null +++ b/intelligence-migrations/0033_tenant_hierarchy.sql @@ -0,0 +1,30 @@ +-- Optional two-level hierarchy; existing accounts and owned records are unchanged. +CREATE TABLE IF NOT EXISTS tenant_organisations ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active', 'deactivated')), + revision INTEGER NOT NULL CHECK (revision >= 1) +); +CREATE TABLE IF NOT EXISTS tenant_teams ( + id TEXT PRIMARY KEY, org_id TEXT NOT NULL, name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active', 'deactivated')), + revision INTEGER NOT NULL CHECK (revision >= 1) +); +CREATE INDEX IF NOT EXISTS tenant_teams_org ON tenant_teams (org_id, id); +CREATE TABLE IF NOT EXISTS tenant_memberships ( + org_id TEXT NOT NULL, principal TEXT NOT NULL, team_id TEXT, + role TEXT NOT NULL CHECK (role IN ('admin', 'billing', 'member')), + status TEXT NOT NULL CHECK (status IN ('active', 'deactivated')), + revision INTEGER NOT NULL CHECK (revision >= 1), + PRIMARY KEY (org_id, principal) +); +CREATE INDEX IF NOT EXISTS tenant_memberships_principal ON tenant_memberships (principal, org_id); +CREATE TABLE IF NOT EXISTS tenant_bindings ( + principal TEXT PRIMARY KEY, org_id TEXT NOT NULL, team_id TEXT, + revision INTEGER NOT NULL CHECK (revision >= 1) +); +CREATE TABLE IF NOT EXISTS tenant_audit ( + id TEXT PRIMARY KEY, actor TEXT NOT NULL, action TEXT NOT NULL, + org_id TEXT, team_id TEXT, principal TEXT, + old_revision INTEGER NOT NULL, new_revision INTEGER NOT NULL, at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS tenant_audit_time ON tenant_audit (at, id); diff --git a/package.json b/package.json index c66e4a8f..9d3f9b48 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_tenant_hierarchy_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/tenants.py b/routes/tenants.py new file mode 100644 index 00000000..668b463d --- /dev/null +++ b/routes/tenants.py @@ -0,0 +1,127 @@ +"""Hierarchy administration uses dashboard sessions; workspace selection uses gateway keys.""" +import re +from functools import wraps + +from flask import g, jsonify, request + +from route_helpers import api_authenticate_only, login_required +from routes.config_snapshots import json_errors +from routes.core import require_admin_dashboard_user +from services import tenant_hierarchy as tenants + + +def _revision(): + value = request.headers.get("If-Match") + if value is None: + raise tenants.TenantError("revision_required", 428) + if not re.fullmatch(r'"[0-9]{1,16}"', value): + raise tenants.TenantError("invalid_revision") + return tenants._revision(int(value[1:-1])) + + +def _data(): + request.max_content_length = tenants.MAX_BODY_BYTES + value = request.get_json(silent=True) + if not isinstance(value, dict): + raise tenants.TenantError("invalid_request") + return value + + +def _reply(value, status=200): + response = jsonify(value) + response.status_code = status + response.headers["Cache-Control"] = "no-store" + if "revision" in value: + response.headers["ETag"] = f'"{value["revision"]}"' + return response + + +def register_tenant_routes(app, *, csrf=None, store=None): + app.extensions["tenant_store"] = store or tenants.TenantStore() + app.register_error_handler(tenants.TenantError, tenants.error_response) + + def is_tenant_route(): + return (request.path == "/admin/organisations" or request.path.startswith("/admin/organisations/") + or request.path == "/v1/workspaces" or request.path.startswith("/v1/workspaces/")) + + def disabled_tenants(): + if is_tenant_route() and not tenants.enabled(): + return jsonify({"error": "not_found"}), 404, {"Cache-Control": "no-store"} + if is_tenant_route(): + request.max_content_length = tenants.MAX_BODY_BYTES + return None + + app.before_request_funcs.setdefault(None, []).insert(0, disabled_tenants) + + @app.after_request + def tenant_no_store(response): + if is_tenant_route(): + response.headers["Cache-Control"] = "no-store" + return response + + def call(operation, **values): + return app.extensions["tenant_store"].request(operation, **values) + + def administrator(view): + @wraps(view) + @json_errors + @login_required + def guarded(*args, **kwargs): + user = require_admin_dashboard_user() + return view(user["username"], *args, **kwargs) + return guarded + + @app.route("/admin/organisations", methods=["GET", "POST"]) + @administrator + def organisations(actor): + if request.method == "GET": + return _reply(call("org_list")) + return _reply(call("org_create", actor=actor, data=_data()), 201) + + @app.route("/admin/organisations/", methods=["GET", "PATCH"]) + @administrator + def organisation(actor, org_id): + if request.method == "GET": + return _reply(call("org_get", org_id=org_id)) + return _reply(call("org_update", actor=actor, org_id=org_id, revision=_revision(), data=_data())) + + @app.route("/admin/organisations//teams", methods=["GET", "POST"]) + @administrator + def teams(actor, org_id): + if request.method == "GET": + return _reply(call("team_list", org_id=org_id)) + return _reply(call("team_create", actor=actor, org_id=org_id, data=_data()), 201) + + @app.patch("/admin/organisations//teams/") + @administrator + def team(actor, org_id, team_id): + return _reply(call("team_update", actor=actor, org_id=org_id, team_id=team_id, + revision=_revision(), data=_data())) + + @app.get("/admin/organisations//members") + @administrator + def members(actor, org_id): + return _reply(call("member_list", org_id=org_id)) + + @app.put("/admin/organisations//members/") + @administrator + def member(actor, org_id, principal): + return _reply(call("member_set", actor=actor, org_id=org_id, principal=principal, + revision=_revision(), data=_data())) + + @app.get("/v1/workspaces") + @api_authenticate_only(required_scope="models") + def workspaces(): + user = g.authenticated_user + return _reply(call("workspaces", principal=user.get("username") or user["id"])) + + @app.post("/v1/workspaces/switch") + @api_authenticate_only(required_scope="models") + def switch_workspace(): + user = g.authenticated_user + principal = user.get("username") or user["id"] + return _reply(call("binding_set", principal=principal, actor=principal, + revision=_revision(), data=_data())) + + # Existing API-key endpoints use this exemption; dashboard mutations retain CSRF. + (csrf or app.extensions.get("csrf")).exempt(switch_workspace) diff --git a/services/auth_service.py b/services/auth_service.py index 9c910254..4b575062 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -943,6 +943,8 @@ def _update_login(cls, username: str, last_login: datetime) -> None: @classmethod def _update_key_usage(cls, username: str, remote_addr: Optional[str] = None) -> None: + from services.tenant_hierarchy import verify_before_key_usage + verify_before_key_usage(username) user = cls._users.get(username) if not user: return @@ -1003,6 +1005,8 @@ def verify_api_key(cls, api_key: Optional[str], remote_addr: Optional[str] = Non and admin_api_key and hmac.compare_digest(api_key, admin_api_key) ): + from services.tenant_hierarchy import verify_before_key_usage + verify_before_key_usage(default_username) return cls._verify_bootstrap_admin(default_username, api_key, remote_addr) if api_key.startswith(KEY_NAMESPACE): diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py index b281f37c..7652d26f 100644 --- a/services/gateway_extensions.py +++ b/services/gateway_extensions.py @@ -7,7 +7,7 @@ from services.provider_catalog_refresh import refresh_provider_catalog_revision AUTHENTICATED_HOOK_ORDER = ( - "request_policy_hook", "prompt_injection_request_hook", "spillover_hook", "pii_request_hook", + "tenant_context_hook", "request_policy_hook", "prompt_injection_request_hook", "spillover_hook", "pii_request_hook", "responses_state_hook", "context_canary_request_hook", "generation_deadline_hook", "latency_slo_request_hook", "idempotency_request_hook", "admit", ) @@ -33,6 +33,14 @@ def after_authentication(): if getattr(g, "gateway_authenticated_hooks_ran", False): return None g.gateway_authenticated_hooks_ran = True + from services.tenant_hierarchy import current_tenant, enabled, tenant_context_hook + # Preserve the default registrar list while resolving legacy identity on every request. + if not enabled(): + g.tenant_context = current_tenant() + elif not any(hook.__name__ == "tenant_context_hook" for hook in current_app.extensions.get("gateway_after_authentication", ())): + refused = tenant_context_hook() + if refused is not None: + return refused for hook in current_app.extensions.get("gateway_after_authentication", ()): refused = hook() if refused is not None: @@ -41,6 +49,13 @@ def after_authentication(): return None +def register_tenants(app): + from services.tenant_hierarchy import TenantError, enabled, error_response, tenant_context_hook + app.register_error_handler(TenantError, error_response) + if enabled(): + register_authenticated_hook(app, tenant_context_hook) + + def register_retention(app): register_authenticated_hook(app, request_policy_hook) @@ -95,7 +110,7 @@ def gateway_callbacks(*, csrf=None): from middleware.rate_limit_headers import register_rate_limit_headers from services.context_canary_hook import register_context_canary from services.pii_redaction import register_pii_redaction - return (register_retention, register_injection_decision, register_batch_spillover, register_pii_redaction, + return (register_tenants, register_retention, register_injection_decision, register_batch_spillover, register_pii_redaction, partial(register_hosted_responses, csrf=csrf), register_context_canary, register_deadline, register_latency_slo, register_managed_idempotency, register_admission, register_cooldown_errors, register_rate_limit_headers) diff --git a/services/tenant_hierarchy.py b/services/tenant_hierarchy.py new file mode 100644 index 00000000..6e1e3db5 --- /dev/null +++ b/services/tenant_hierarchy.py @@ -0,0 +1,366 @@ +"""Server-owned tenant bindings and atomic, content-free hierarchy administration.""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import queue +import re +import sqlite3 +import threading +import time +import uuid +from contextlib import closing +from datetime import datetime, timezone +from functools import lru_cache + +from flask import current_app, g, has_request_context, jsonify + +from services.enterprise_contract import AuthorityOperation, TenantContext, legacy_tenant +from services.sqlite_store import connect, storage_path + +logger = logging.getLogger(__name__) +PRIVATE_URL = "http://intelligence.internal/v1/managed-state/tenants" +MAX_BODY_BYTES = 8192 +_SLOTS = threading.BoundedSemaphore(8) +_ID = re.compile(r"[A-Za-z0-9_:.\-]{1,128}\Z") +OPERATIONS = frozenset({"org_list", "org_get", "org_create", "org_update", "team_list", "team_create", + "team_update", "member_list", "member_set", "workspaces", "binding_set", "resolve"}) +ERROR_STATUSES = {"workspace_not_found": 404, "workspace_forbidden": 403, "principal_not_found": 404, + "revision_required": 428, "revision_conflict": 412, "tenant_limit_reached": 409, + "invalid_request": 400, "invalid_revision": 400, "tenant_storage_unavailable": 503} + + +class TenantError(Exception): + def __init__(self, code, status=400): + super().__init__(code) + self.code, self.status = code, status + + +def error_response(error): + return jsonify({"error": {"code": error.code, "message": "The workspace operation could not be authorized or stored."}}), error.status, {"Cache-Control": "no-store"} + + +@lru_cache(maxsize=1) +def _warn_once(): + logger.warning("Invalid ORGANISATIONS_ENABLED; organisations disabled") + + +def enabled(): + flag = os.environ.get("ORGANISATIONS_ENABLED", "").strip().lower() + if flag in {"", "0", "false", "no", "off"}: + return False + if flag in {"1", "true", "yes", "on"}: + return True + _warn_once() + return False + + +def _text(value, maximum=128): + if not isinstance(value, str) or not value.strip() or len(value) > maximum or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise TenantError("invalid_request") + return value + + +def _identifier(value): + if not isinstance(value, str) or not _ID.fullmatch(value): + raise TenantError("invalid_request") + return value + + +def _revision(value): + if value is None: + raise TenantError("revision_required", 428) + if type(value) is not int or not 0 <= value < 9007199254740000: + raise TenantError("invalid_revision") + return value + + +def _legacy(principal): + try: + context = TenantContext(principal) + except ValueError: + context = TenantContext("principal:" + hashlib.sha256(principal.encode()).hexdigest()) + return legacy_tenant(AuthorityOperation(context, "workspace", 0, "resolve")) + + +def private_call(document): + """Reuse the bounded private transport, with one submission and no replay.""" + from services import intelligence_d1_store as transport + try: + body = json.dumps(document, separators=(",", ":"), allow_nan=False).encode() + except (ValueError, TypeError): + raise TenantError("invalid_request") from None + if len(body) > MAX_BODY_BYTES or not _SLOTS.acquire(blocking=False): + raise TenantError("tenant_storage_unavailable", 503) + stopped, results = threading.Event(), queue.Queue(maxsize=1) + deadline = time.monotonic() + 5 + thread = threading.Thread(target=transport._submit, + args=(PRIVATE_URL, body, stopped, deadline, results, _SLOTS, (200,), (2, 3)), daemon=True) + try: + thread.start() + except RuntimeError: + _SLOTS.release() + raise TenantError("tenant_storage_unavailable", 503) from None + try: + success, result = results.get(timeout=max(0, deadline - time.monotonic())) + if not success: + if isinstance(result, transport.PrivateIntelligenceError): + raise TenantError(result.code, result.status) + raise TenantError("tenant_storage_unavailable", 503) + return result + except queue.Empty: + raise TenantError("tenant_storage_unavailable", 503) from None + finally: + stopped.set() + + +def _principal_exists(principal): + from services.auth_service import AuthService + return AuthService._load_user_by_username(principal) is not None + + +class TenantStore: + def __init__(self, *, call=None, principal_exists=None): + self.call = call or private_call + self.principal_exists = principal_exists or _principal_exists + + def request(self, operation, **values): + if not enabled(): + raise TenantError("not_found", 404) + if operation not in OPERATIONS: + raise TenantError("invalid_request") + if os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip() == "d1": + return self._remote(operation, values) + try: + path = storage_path("AUTH_DB_PATH", "auth.sqlite3") + if not path.is_file(): + raise TenantError("tenant_storage_unavailable", 503) + with closing(connect(path)) as db: + db.execute("BEGIN IMMEDIATE" if operation.endswith(("create", "update", "set")) else "BEGIN") + # Verify all tables and columns before a mutation or a legacy resolution. + for sql in ("SELECT id,name,status,revision FROM tenant_organisations LIMIT 0", + "SELECT id,org_id,name,status,revision FROM tenant_teams LIMIT 0", + "SELECT org_id,principal,team_id,role,status,revision FROM tenant_memberships LIMIT 0", + "SELECT principal,org_id,team_id,revision FROM tenant_bindings LIMIT 0", + "SELECT id,actor,action,org_id,team_id,principal,old_revision,new_revision,at FROM tenant_audit LIMIT 0"): + db.execute(sql) + result = self._execute(db, operation, values) + db.commit() + return result + except TenantError: + raise + except Exception: + raise TenantError("tenant_storage_unavailable", 503) from None + + def _remote(self, operation, values): + try: + document = self.call({"version": 1, "operation": operation, **values}) + if not isinstance(document, dict) or document.get("version") != 1: + raise ValueError() + if "error" in document: + code, status = document["error"]["code"], document["status"] + if ERROR_STATUSES.get(code) != status: + raise ValueError() + raise TenantError(code, status) + result = document["result"] + if not isinstance(result, dict): + raise ValueError() + return result + except TenantError: + raise + except Exception: + raise TenantError("tenant_storage_unavailable", 503) from None + + @staticmethod + def _one(db, sql, params=()): + row = db.execute(sql, params).fetchone() + return dict(row) if row else None + + def _org(self, db, org_id, *, active=False): + row = self._one(db, "SELECT * FROM tenant_organisations WHERE id=?", (_identifier(org_id),)) + if row is None: + raise TenantError("workspace_not_found", 404) + if active and row["status"] != "active": + raise TenantError("workspace_forbidden", 403) + return row + + def _team(self, db, org_id, team_id, *, active=False): + row = self._one(db, "SELECT * FROM tenant_teams WHERE org_id=? AND id=?", (org_id, _identifier(team_id))) + if row is None: + raise TenantError("workspace_not_found", 404) + if active and row["status"] != "active": + raise TenantError("workspace_forbidden", 403) + return row + + def _workspace(self, db, principal, org_id, team_id): + org = self._org(db, org_id, active=True) + team = self._team(db, org_id, team_id, active=True) if team_id is not None else None + membership = self._one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", (org_id, principal)) + if not membership or membership["status"] != "active" or membership["team_id"] != team_id: + raise TenantError("workspace_forbidden", 403) + return org["revision"] + (team["revision"] if team else 0) + membership["revision"] + + @staticmethod + def _audit(db, actor, action, org_id, team_id, principal, old, new): + db.execute("INSERT INTO tenant_audit VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (uuid.uuid4().hex, _text(actor), action, org_id, team_id, principal, old, new, + datetime.now(timezone.utc).isoformat())) + + @staticmethod + def _cas(row, revision): + if (row["revision"] if row else 0) != _revision(revision): + raise TenantError("revision_conflict", 412) + + def _execute(self, db, operation, v): + if operation == "org_list": + return {"organisations": [dict(row) for row in db.execute("SELECT * FROM tenant_organisations ORDER BY id")]} + if operation in {"workspaces", "resolve", "binding_set"}: + return self._principal_operation(db, operation, v) + if operation == "org_create": + return self._named_write(db, operation, v, None) + org_id = v.get("org_id") + org = self._org(db, org_id) + if operation == "org_get": + return org + if operation in {"org_update", "team_create", "team_update"}: + return self._named_write(db, operation, v, org) + if operation == "team_list": + return {"teams": [dict(row) for row in db.execute("SELECT * FROM tenant_teams WHERE org_id=? ORDER BY id", (org_id,))]} + if operation == "member_list": + return {"members": [dict(row) for row in db.execute("SELECT * FROM tenant_memberships WHERE org_id=? ORDER BY principal", (org_id,))]} + return self._member_write(db, v) + + def _named_write(self, db, operation, v, org): + data = v.get("data") + creating, is_team = operation.endswith("create"), operation.startswith("team") + if not isinstance(data, dict) or not data or set(data) - {"name", "status"}: + raise TenantError("invalid_request") + if creating and ("name" not in data or data.get("status", "active") != "active"): + raise TenantError("invalid_request") + if is_team and creating: + self._org(db, v["org_id"], active=True) + if db.execute("SELECT count(*) FROM tenant_teams WHERE org_id=?", (v["org_id"],)).fetchone()[0] >= 100: + raise TenantError("tenant_limit_reached", 409) + row = None if creating else self._team(db, v["org_id"], v["team_id"]) if is_team else org + if not creating: + self._cas(row, v.get("revision")) + name = _text(data.get("name", row["name"] if row else None), 200) + status = data.get("status", row["status"] if row else "active") + if not isinstance(status, str) or status not in {"active", "deactivated"}: + raise TenantError("invalid_request") + identifier = uuid.uuid4().hex if creating else row["id"] + old, new = (row["revision"] if row else 0), (row["revision"] + 1 if row else 1) + if is_team: + db.execute("INSERT INTO tenant_teams VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,status=excluded.status,revision=excluded.revision", + (identifier, v["org_id"], name, status, new)) + else: + db.execute("INSERT INTO tenant_organisations VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,status=excluded.status,revision=excluded.revision", + (identifier, name, status, new)) + org_id, team_id = (v["org_id"], identifier) if is_team else (identifier, None) + self._audit(db, v["actor"], operation, org_id, team_id, None, old, new) + return self._team(db, org_id, team_id) if is_team else self._org(db, org_id) + + def _member_write(self, db, v): + org_id, principal, data = v["org_id"], _text(v.get("principal")), v.get("data") + if not isinstance(data, dict) or not data or set(data) - {"role", "team_id", "status", "bind", "binding_revision"}: + raise TenantError("invalid_request") + if not self.principal_exists(principal): + raise TenantError("principal_not_found", 404) + row = self._one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", (org_id, principal)) + self._cas(row, v.get("revision")) + if row is None and db.execute("SELECT count(*) FROM tenant_memberships WHERE org_id=?", (org_id,)).fetchone()[0] >= 1000: + raise TenantError("tenant_limit_reached", 409) + team_id = data.get("team_id", row["team_id"] if row else None) + role, status = data.get("role", row["role"] if row else "member"), data.get("status", row["status"] if row else "active") + if not isinstance(role, str) or role not in {"admin", "billing", "member"} or not isinstance(status, str) or status not in {"active", "deactivated"}: + raise TenantError("invalid_request") + if status == "active": + self._org(db, org_id, active=True) + if team_id is not None: + self._team(db, org_id, team_id, active=status == "active") + if "bind" in data and type(data["bind"]) is not bool: + raise TenantError("invalid_request") + if "binding_revision" in data and not data.get("bind"): + raise TenantError("invalid_request") + old = row["revision"] if row else 0 + db.execute("INSERT INTO tenant_memberships VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(org_id,principal) DO UPDATE SET team_id=excluded.team_id,role=excluded.role,status=excluded.status,revision=excluded.revision", + (org_id, principal, team_id, role, status, old + 1)) + self._audit(db, v["actor"], "member_set", org_id, team_id, principal, old, old + 1) + if data.get("bind"): + self._binding_write(db, principal, v["actor"], data.get("binding_revision"), {"org_id": org_id, "team_id": team_id}) + return self._one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", (org_id, principal)) + + def _principal_operation(self, db, operation, v): + principal = _text(v.get("principal")) + binding = self._one(db, "SELECT * FROM tenant_bindings WHERE principal=?", (principal,)) + if operation == "binding_set": + if v.get("actor") != principal: + raise TenantError("workspace_forbidden", 403) + return self._binding_write(db, principal, principal, v.get("revision"), v.get("data")) + if operation == "workspaces": + rows = db.execute("SELECT m.* FROM tenant_memberships m JOIN tenant_organisations o ON o.id=m.org_id LEFT JOIN tenant_teams t ON t.id=m.team_id AND t.org_id=m.org_id WHERE m.principal=? AND m.status='active' AND o.status='active' AND (m.team_id IS NULL OR t.status='active') ORDER BY m.org_id", (principal,)) + return {"memberships": [dict(row) for row in rows], "binding": binding or {"principal": principal, "org_id": None, "team_id": None, "revision": 0}} + if binding is None: + return {"principal_id": _legacy(principal).principal_id, "org_id": None, "team_id": None, "grants_revision": 0} + revision = self._workspace(db, principal, binding["org_id"], binding["team_id"]) + return {"principal_id": _legacy(principal).principal_id, "org_id": binding["org_id"], "team_id": binding["team_id"], "grants_revision": revision + binding["revision"]} + + def _binding_write(self, db, principal, actor, revision, data): + if not isinstance(data, dict) or set(data) != {"org_id", "team_id"}: + raise TenantError("invalid_request") + self._workspace(db, principal, data["org_id"], data["team_id"]) + row = self._one(db, "SELECT * FROM tenant_bindings WHERE principal=?", (principal,)) + self._cas(row, revision) + old = row["revision"] if row else 0 + db.execute("INSERT INTO tenant_bindings VALUES (?, ?, ?, ?) ON CONFLICT(principal) DO UPDATE SET org_id=excluded.org_id,team_id=excluded.team_id,revision=excluded.revision", + (principal, data["org_id"], data["team_id"], old + 1)) + self._audit(db, actor, "binding_set", data["org_id"], data["team_id"], principal, old, old + 1) + return self._one(db, "SELECT * FROM tenant_bindings WHERE principal=?", (principal,)) + + +def resolve_principal(principal, *, store=None): + legacy = _legacy(principal) + if not enabled(): + return legacy + value = (store or TenantStore()).request("resolve", principal=principal) + try: + context = TenantContext(**value) + if context.principal_id != legacy.principal_id: + raise ValueError() + return context + except (ValueError, TypeError): + raise TenantError("tenant_storage_unavailable", 503) from None + + +def current_tenant(): + if has_request_context() and hasattr(g, "tenant_context"): + return g.tenant_context + user = getattr(g, "authenticated_user", {}) if has_request_context() else {} + return _legacy(user.get("username") or user.get("id") or "unknown") + + +def tenant_namespace(context=None): + context = context or current_tenant() + if context.org_id is None: + return "" + return "org:" + context.org_id + ("/team:" + context.team_id if context.team_id else "") + + +def tenant_context_hook(): + user = getattr(g, "authenticated_user", {}) + principal = user.get("username") or user.get("id") or "unknown" + try: + # Verified context was resolved just before usage metadata in authentication. + verified = getattr(g, "verified_tenant", None) + g.tenant_context = verified if verified and verified.principal_id == _legacy(principal).principal_id else resolve_principal( + principal, store=current_app.extensions.get("tenant_store")) + except TenantError as error: + return error_response(error) + return None + + +def verify_before_key_usage(principal): + if enabled() and has_request_context(): + g.verified_tenant = resolve_principal(principal, store=current_app.extensions.get("tenant_store")) diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..c46a2612 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -178,6 +178,16 @@ "app.get('/status.json')", "app.route('/v1/health/checks', methods=['POST', 'OPTIONS'])" ], + "routes/tenants.py": [ + "app.get('/admin/organisations//members')", + "app.get('/v1/workspaces')", + "app.patch('/admin/organisations//teams/')", + "app.post('/v1/workspaces/switch')", + "app.put('/admin/organisations//members/')", + "app.route('/admin/organisations', methods=['GET', 'POST'])", + "app.route('/admin/organisations/', methods=['GET', 'PATCH'])", + "app.route('/admin/organisations//teams', methods=['GET', 'POST'])" + ], "routes/unified.py": [ "app.route('/admin/models//disable', methods=['POST'])", "app.route('/v1/chat/completions', methods=['POST', 'OPTIONS'])", @@ -409,6 +419,7 @@ "worker/sse-heartbeat.mjs", "worker/status-page.mjs", "worker/stream-cost-breaker.mjs", + "worker/tenants-d1.mjs", "worker/upstream-cancellation.mjs", "worker/usage-buckets-d1.mjs", "worker/usage-ledger-d1.mjs", diff --git a/tests/test_tenant_hierarchy.py b/tests/test_tenant_hierarchy.py new file mode 100644 index 00000000..535ee815 --- /dev/null +++ b/tests/test_tenant_hierarchy.py @@ -0,0 +1,353 @@ +"""Registered tenant boundaries and atomic hierarchy storage with synthetic accounts.""" +import importlib +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from flask import Flask, g, jsonify +from flask_wtf.csrf import CSRFProtect + +from services import tenant_hierarchy as th + + +@pytest.fixture +def store(monkeypatch, tmp_path): + monkeypatch.setenv("ORGANISATIONS_ENABLED", "true") + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "") + monkeypatch.setenv("CONTROL_PLANE_DATABASE_URL", "") + path = tmp_path / "auth.sqlite3" + monkeypatch.setenv("AUTH_DB_PATH", str(path)) + with sqlite3.connect(path) as db: + db.executescript((Path(__file__).resolve().parents[1] / "intelligence-migrations/0033_tenant_hierarchy.sql").read_text()) + db.execute("CREATE TABLE old_records (owner TEXT)") + db.execute("INSERT INTO old_records VALUES ('alice')") + return th.TenantStore(principal_exists=lambda principal: principal in {"alice", "bob", "operator"}) + + +def hierarchy(store): + org = store.request("org_create", actor="operator", data={"name": "Example"}) + team = store.request("team_create", actor="operator", org_id=org["id"], data={"name": "Support"}) + return org, team + + +def member(store, org, team=None, **data): + return store.request("member_set", actor="operator", org_id=org["id"], principal="alice", revision=0, + data={"role": "member", "team_id": team["id"] if team else None, "status": "active", **data}) + + +def test_disabled_is_legacy_and_does_not_touch_store(monkeypatch): + for flag in ("", "false", "malformed-secret-value"): + monkeypatch.setenv("ORGANISATIONS_ENABLED", flag) + fake = Mock(side_effect=AssertionError("No storage")) + context = th.resolve_principal("alice", store=Mock(request=fake)) + assert context == th.TenantContext("alice") + assert th.tenant_namespace(context) == "" + fake.assert_not_called() + + +def test_flag_warns_once_without_value(monkeypatch, caplog): + th._warn_once.cache_clear() + monkeypatch.setenv("ORGANISATIONS_ENABLED", "private-bad-value") + assert not th.enabled() and not th.enabled() + assert caplog.text.count("Invalid ORGANISATIONS_ENABLED") == 1 + assert "private-bad-value" not in caplog.text + + +def test_ids_are_generated_and_schema_is_additive(store): + org, team = hierarchy(store) + assert org["id"] != team["id"] and org["revision"] == team["revision"] == 1 + with pytest.raises(th.TenantError) as error: + store.request("org_create", actor="operator", data={"name": "Other", "id": org["id"]}) + assert error.value.status == 400 + with sqlite3.connect(th.storage_path("AUTH_DB_PATH", "auth.sqlite3")) as db: + assert db.execute("SELECT owner FROM old_records").fetchone()[0] == "alice" + assert db.execute("SELECT count(*) FROM tenant_bindings").fetchone()[0] == 0 + + +def test_binding_is_explicit_and_membership_isolated(store): + org, team = hierarchy(store) + member(store, org, team) + assert th.resolve_principal("alice", store=store).org_id is None + assert store.request("workspaces", principal="bob")["memberships"] == [] + binding = store.request("binding_set", actor="alice", principal="alice", revision=0, + data={"org_id": org["id"], "team_id": team["id"]}) + assert binding["revision"] == 1 + context = th.resolve_principal("alice", store=store) + assert th.tenant_namespace(context) == f"org:{org['id']}/team:{team['id']}" + assert store.request("workspaces", principal="alice")["binding"] == binding + with pytest.raises(th.TenantError, match="workspace_forbidden"): + store.request("binding_set", actor="bob", principal="bob", revision=0, + data={"org_id": org["id"], "team_id": team["id"]}) + + +def test_foreign_team_missing_and_revoked_binding_fail_closed(store): + org, team = hierarchy(store) + other, foreign = hierarchy(store) + with pytest.raises(th.TenantError, match="workspace_not_found"): + member(store, org, foreign) + member(store, org, team, bind=True, binding_revision=0) + before = th.resolve_principal("alice", store=store) + store.request("member_set", actor="operator", org_id=org["id"], principal="alice", revision=1, + data={"status": "deactivated"}) + with pytest.raises(th.TenantError, match="workspace_forbidden"): + th.resolve_principal("alice", store=store) + store.request("member_set", actor="operator", org_id=org["id"], principal="alice", revision=2, + data={"status": "active"}) + assert th.resolve_principal("alice", store=store).grants_revision > before.grants_revision + store.request("team_update", actor="operator", org_id=org["id"], team_id=team["id"], revision=1, + data={"status": "deactivated"}) + with pytest.raises(th.TenantError, match="workspace_forbidden"): + th.resolve_principal("alice", store=store) + with sqlite3.connect(th.storage_path("AUTH_DB_PATH", "auth.sqlite3")) as db: + db.execute("UPDATE tenant_bindings SET team_id='missing' WHERE principal='alice'") + with pytest.raises(th.TenantError, match="workspace_not_found"): + th.resolve_principal("alice", store=store) + + +def test_cas_one_winner_and_content_free_audit(store): + org, _ = hierarchy(store) + def write(name): + try: + return store.request("org_update", actor="operator", org_id=org["id"], revision=1, data={"name": name}) + except th.TenantError as error: + return error.status + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(write, ["confidential-one", "confidential-two"])) + assert sum(isinstance(result, dict) for result in results) == 1 and 412 in results + with sqlite3.connect(th.storage_path("AUTH_DB_PATH", "auth.sqlite3")) as db: + rows = db.execute("SELECT * FROM tenant_audit").fetchall() + assert len(rows) == 3 + assert "confidential" not in repr(rows) + + +def test_caps_and_unknown_principals(store): + org, team = hierarchy(store) + with sqlite3.connect(th.storage_path("AUTH_DB_PATH", "auth.sqlite3")) as db: + db.executemany("INSERT INTO tenant_teams VALUES (?, ?, ?, 'deactivated', 1)", + [(f"team_{i}", org["id"], "Example") for i in range(99)]) + db.executemany("INSERT INTO tenant_memberships VALUES (?, ?, NULL, 'member', 'deactivated', 1)", + [(org["id"], f"principal_{i}") for i in range(1000)]) + with pytest.raises(th.TenantError, match="tenant_limit_reached"): + store.request("team_create", actor="operator", org_id=org["id"], data={"name": "Extra"}) + with pytest.raises(th.TenantError, match="tenant_limit_reached"): + member(store, org) + with pytest.raises(th.TenantError, match="principal_not_found"): + store.request("member_set", actor="operator", org_id=org["id"], principal="unknown", revision=0, + data={"role": "member"}) + + +@pytest.fixture +def registered(store, monkeypatch): + routes = importlib.import_module("routes.tenants") + registrar = importlib.import_module("services.gateway_extensions") + app = Flask(__name__) + app.config.update(TESTING=True, SECRET_KEY="synthetic-tenant-session", WTF_CSRF_ENABLED=False) + csrf = CSRFProtect(app) + routes.register_tenant_routes(app, csrf=csrf, store=store) + registrar.register_tenants(app) + auth = importlib.import_module("services.auth_service").AuthService + monkeypatch.setattr(auth, "is_authenticated", lambda: True) + monkeypatch.setattr(auth, "get_current_user", lambda: {"username": "operator", "is_admin": True}) + monkeypatch.setattr(auth, "verify_api_key", lambda *_: {"username": "alice", "scopes": ["models", "chat"]}) + for guard in (routes.login_required, routes.api_authenticate_only, routes.require_admin_dashboard_user): + monkeypatch.setitem(guard.__globals__, "AuthService", auth) + monkeypatch.setattr(routes.api_authenticate_only.__globals__["request_accounting"], + "check_key_controls", lambda *_: None) + @app.get("/probe") + def probe(): + g.authenticated_user = {"username": "alice"} + refused = registrar.after_authentication() + return refused if refused is not None else jsonify(namespace=th.tenant_namespace()) + return app, app.test_client(), auth, registrar + + +def test_registered_disabled_routes_precede_auth_csrf_and_json(registered, monkeypatch): + app, client, auth, _ = registered + app.config["WTF_CSRF_ENABLED"] = True + monkeypatch.setattr(auth, "is_authenticated", Mock(side_effect=AssertionError("No authentication"))) + monkeypatch.setenv("ORGANISATIONS_ENABLED", "") + for path in ("/admin/organisations", "/admin/organisations/absent", "/admin/organisations/absent/teams", + "/admin/organisations/absent/teams/absent", "/admin/organisations/absent/members", + "/admin/organisations/absent/members/alice", "/v1/workspaces", "/v1/workspaces/switch"): + for method in ("GET", "POST", "PATCH", "PUT"): + result = client.open(path, method=method, data="not json") + assert result.status_code == 404 and result.is_json + assert result.headers["Cache-Control"] == "no-store" + + +def test_registered_admin_cas_guards_and_csrf(registered, monkeypatch): + app, client, auth, _ = registered + org = client.post("/admin/organisations", json={"name": "Example"}).json + assert org["revision"] == 1 + path = f"/admin/organisations/{org['id']}" + assert client.patch(path, json={"name": "Next"}).status_code == 428 + assert client.patch(path, json={"name": "Next"}, headers={"If-Match": '"0"'}).status_code == 412 + response = client.patch(path, json={"name": "Next"}, headers={"If-Match": '"1"'}) + assert response.json["revision"] == 2 and response.headers["ETag"] == '"2"' + monkeypatch.setattr(auth, "get_current_user", lambda: {"username": "operator", "is_admin": False}) + assert client.get(path).status_code == 403 + app.config["WTF_CSRF_ENABLED"] = True + assert client.post("/admin/organisations", json={"name": "Example"}).status_code == 400 + + +def test_workspace_switch_and_spoofing_registered(registered, store): + _, client, _, registrar = registered + org, team = hierarchy(store) + member(store, org, team) + headers = {"Authorization": "Bearer synthetic", "X-Org": org["id"], "X-Team": team["id"], + "X-MultiLLM-Workspace": org["id"]} + assert client.get("/probe", headers=headers).json == {"namespace": ""} + assert client.get("/v1/workspaces", headers=headers).json["memberships"][0]["org_id"] == org["id"] + response = client.post("/v1/workspaces/switch", headers={**headers, "If-Match": '"0"'}, + json={"org_id": org["id"], "team_id": team["id"]}) + assert response.status_code == 200 + assert client.get("/probe", headers={"X-Org": "foreign"}).json["namespace"] == th.tenant_namespace(th.resolve_principal("alice", store=store)) + assert registrar.AUTHENTICATED_HOOK_ORDER[0] == "tenant_context_hook" + + +def test_missing_schema_and_private_errors_are_json_503(registered, monkeypatch, tmp_path): + _, client, _, _ = registered + monkeypatch.setenv("AUTH_DB_PATH", str(tmp_path / "missing.sqlite3")) + response = client.get("/probe") + assert response.status_code == 503 and response.json["error"]["code"] == "tenant_storage_unavailable" + monkeypatch.setenv("INTELLIGENCE_STORAGE_BACKEND", "d1") + fake = Mock(return_value={"version": 1, "error": {"code": "workspace_forbidden"}, "status": 403}) + remote = th.TenantStore(call=fake) + with pytest.raises(th.TenantError, match="workspace_forbidden"): + th.resolve_principal("alice", store=remote) + assert fake.call_count == 1 + + +def test_registered_tenant_failure_stops_later_hooks_and_metadata(registered, store, monkeypatch): + app, client, auth, registrar = registered + org, team = hierarchy(store) + member(store, org, team, bind=True, binding_revision=0) + store.request("org_update", actor="operator", org_id=org["id"], revision=1, data={"status": "deactivated"}) + later = Mock(side_effect=AssertionError("No downstream storage or provider")) + later.__name__ = "request_policy_hook" + registrar.register_authenticated_hook(app, later) + response = client.get("/probe") + assert response.status_code == 403 and response.json["error"]["code"] == "workspace_forbidden" + later.assert_not_called() + monkeypatch.setattr(auth, "_users", {"alice": {"last_used_at": None}}) + with app.test_request_context("/probe"): + with pytest.raises(th.TenantError, match="workspace_forbidden"): + auth._update_key_usage("alice") + assert auth._users["alice"]["last_used_at"] is None + + +def test_binding_cas_and_admin_binding_are_atomic(store): + org, team = hierarchy(store) + member(store, org, team, bind=True, binding_revision=0) + with pytest.raises(th.TenantError, match="revision_conflict"): + store.request("member_set", actor="operator", org_id=org["id"], principal="alice", revision=1, + data={"role": "billing", "bind": True, "binding_revision": 0}) + assert store.request("member_list", org_id=org["id"])["members"][0]["role"] == "member" + with pytest.raises(th.TenantError, match="revision_conflict"): + store.request("binding_set", actor="alice", principal="alice", revision=0, + data={"org_id": org["id"], "team_id": team["id"]}) + + +def test_create_app_mounts_routes_with_no_live_side_effects(store, monkeypatch): + for name, value in {"FLASK_SECRET_KEY": "synthetic-tenant-session", "JWT_SECRET": "synthetic-tenant-jwt", + "ADMIN_API_KEY": "synthetic-tenant-admin", "CONFIG_REVISION_SYNC_ENABLED": "false"}.items(): + monkeypatch.setenv(name, value) + with patch("config.load_runtime_env"), patch("env_loader.load_runtime_env"), \ + patch("requests.sessions.Session.send", side_effect=AssertionError("No network")): + module = importlib.import_module("app") + with patch.object(module, "load_runtime_env"), patch.object(module.AuthService, "initialize"), patch.object(module.usage_ledger, "start"): + app = module.create_app() + app.config.update(TESTING=True, WTF_CSRF_ENABLED=False) + app.extensions["tenant_store"] = store + with patch.object(module.AuthService, "is_authenticated", return_value=True), \ + patch.object(module.AuthService, "get_current_user", return_value={"username": "operator", "is_admin": True}): + response = app.test_client().post("/admin/organisations", json={"name": "Registered"}) + assert response.status_code == 201 + assert app.extensions["gateway_after_authentication"][0].__name__ == "tenant_context_hook" + + +@pytest.mark.parametrize("data", [{"role": None}, {"role": ["admin"]}, {"status": None}, {"status": ["active"]}, {"bind": "true"}]) +def test_membership_invalid_types_are_rejected_without_mutation(store, data): + org, _ = hierarchy(store) + with pytest.raises(th.TenantError) as error: + member(store, org, **data) + assert error.value.status == 400 + assert store.request("member_list", org_id=org["id"])["members"] == [] + + +def test_registered_full_admin_lifecycle_and_workspace_scope(registered, store, monkeypatch): + _, client, auth, _ = registered + org = client.post("/admin/organisations", json={"name": "Example"}).json + base = f"/admin/organisations/{org['id']}" + team = client.post(base + "/teams", json={"name": "Support"}).json + assert client.get(base + "/teams").json["teams"] == [team] + response = client.put(base + "/members/alice", headers={"If-Match": '\"0\"'}, + json={"role": "billing", "team_id": team["id"], "status": "active"}) + assert response.status_code == 200 + assert client.get(base + "/members").json["members"] == [response.json] + assert client.patch(base + f"/teams/{team['id']}", headers={"If-Match": '\"1\"'}, json={"name": "Renamed"}).status_code == 200 + assert client.post("/v1/workspaces/switch", headers={"Authorization": "Bearer synthetic"}, + json={"org_id": org["id"], "team_id": team["id"]}).status_code == 428 + monkeypatch.setattr(auth, "verify_api_key", lambda *_: {"username": "bob", "scopes": ["chat"]}) + assert client.get("/v1/workspaces", headers={"Authorization": "Bearer synthetic"}).status_code == 403 + + +def test_account_tenant_check_follows_matching_hash_and_cached_verification(monkeypatch): + module = importlib.import_module("services.auth_service") + auth = module.AuthService + key = "shared01-synthetic-matching" + users = {"alice": {"api_key_hash": "other", "revoked_at": None}, + "bob": {"api_key_hash": "matching", "revoked_at": None}} + checks = [] + monkeypatch.setenv("ADMIN_API_KEY", "") + monkeypatch.setattr(auth, "_users", users) + monkeypatch.setattr(auth, "_verified_keys", {}) + monkeypatch.setattr(auth, "_load_users_by_api_key_prefix", lambda _: list(users.items())) + monkeypatch.setattr(auth, "_public_user", lambda username, _: {"username": username}) + monkeypatch.setattr(module.key_controls, "verify_revisioned_key", lambda *_: module.key_controls.LEGACY_AUTH) + monkeypatch.setattr(module.user_store, "using_d1", lambda: True) + touch = Mock() + monkeypatch.setattr(module.user_store, "touch_user", touch) + + def check_hash(stored, provided): + checks.append(("hash", stored, provided)) + return stored == "matching" and provided == key + + def verify_tenant(principal): + checks.append(("tenant", principal)) + assert principal == "bob" + + monkeypatch.setattr(module, "check_password_hash", check_hash) + monkeypatch.setattr(th, "verify_before_key_usage", verify_tenant) + assert auth.verify_api_key(key) == {"username": "bob"} + assert checks == [("hash", "other", key), ("hash", "matching", key), ("tenant", "bob")] + checks.clear() + assert auth.verify_api_key(key) == {"username": "bob"} + assert checks == [("tenant", "bob")] + checks.clear() + assert auth.verify_api_key("shared01-wrong-key") is None + assert all(check[0] == "hash" for check in checks) + assert touch.call_count == 1 + assert touch.call_args.args[0] == "bob" + + +def test_bootstrap_tenant_check_follows_environment_key_verification(monkeypatch): + module = importlib.import_module("services.auth_service") + auth = module.AuthService + monkeypatch.setenv("ADMIN_USERNAME", "operator") + monkeypatch.setenv("ADMIN_API_KEY", "synthetic-bootstrap") + monkeypatch.setattr(auth, "_verified_keys", {}) + monkeypatch.setattr(auth, "_load_users_by_api_key_prefix", lambda _: []) + monkeypatch.setattr(module.key_controls, "verify_revisioned_key", lambda *_: module.key_controls.LEGACY_AUTH) + tenant = Mock() + verified = Mock(return_value={"username": "operator"}) + monkeypatch.setattr(th, "verify_before_key_usage", tenant) + monkeypatch.setattr(auth, "_verify_bootstrap_admin", verified) + assert auth.verify_api_key("synthetic-bootstrap-wrong") is None + tenant.assert_not_called() + verified.assert_not_called() + assert auth.verify_api_key("synthetic-bootstrap") == {"username": "operator"} + tenant.assert_called_once_with("operator") + verified.assert_called_once_with("operator", "synthetic-bootstrap", None) diff --git a/tests/test_tenant_hierarchy_worker.mjs b/tests/test_tenant_hierarchy_worker.mjs new file mode 100644 index 00000000..a3456599 --- /dev/null +++ b/tests/test_tenant_hierarchy_worker.mjs @@ -0,0 +1,341 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { DatabaseSync } from "node:sqlite"; +import { readFileSync } from "node:fs"; +import { scryptSync } from "node:crypto"; +import { handleKnowledgeEdgeRequest } from "../worker/knowledge-edge.mjs"; +import { handleRealtimeRequest } from "../worker/realtime.mjs"; +import { activeUsersByPrefix, USER_FIELDS } from "../worker/control-users-d1.mjs"; +import { handleTenantRequest, organisationsEnabled, resolveAccountTenant, tenantNamespace } from "../worker/tenants-d1.mjs"; +import { handleManagedStateRequest } from "../worker/managed-state-dispatch.mjs"; +import { nativeGenerationFetch } from "../worker/gateway-extensions.mjs"; + +test("disabled tenant domain and resolver touch no storage", async () => { + const db = { prepare() { throw Error("No storage"); } }; + for (const flag of ["", "false", "invalid-private-value"]) { + const env = { ORGANISATIONS_ENABLED: flag, INTELLIGENCE_DB: db }; + assert.equal(organisationsEnabled(env, () => {}), false); + assert.equal(tenantNamespace(await resolveAccountTenant(db, "alice", env)), ""); + const response = await handleManagedStateRequest(new Request("http://intelligence.internal/v1/managed-state/tenants", { method: "POST", body: "broken" }), env); + assert.equal(response.status, 404); + assert.equal(response.headers.get("cache-control"), "no-store"); + } +}); + +test("tenant domain is private, bounded and fails closed on missing schema", async () => { + const env = { ORGANISATIONS_ENABLED: "true", INTELLIGENCE_DB: { prepare() { throw Error("no such table: sensitive"); } } }; + const send = (url, body) => handleTenantRequest(new Request(url, { method: "POST", headers: { "content-type": "application/json" }, body }), env); + assert.equal((await send("https://public.example/v1/managed-state/tenants", "{}")).status, 404); + assert.equal((await send("http://intelligence.internal/v1/managed-state/tenants", "x".repeat(9000))).status, 400); + const response = await send("http://intelligence.internal/v1/managed-state/tenants", JSON.stringify({ version: 1, operation: "resolve", principal: "alice" })); + assert.equal(response.status, 503); + assert.equal((await response.json()).error.code, "tenant_storage_unavailable"); +}); + +test("native generation rejects unavailable tenant authority before provider or hooks", async () => { + let calls = 0; + const result = await nativeGenerationFetch(new Request("https://gateway.example/v1/chat/completions", { method: "POST", body: JSON.stringify({ org_id: "spoofed" }), headers: { "X-Org": "spoofed" } }), + { ORGANISATIONS_ENABLED: "true" }, {}, { authenticated: true, principal: { id: "alice" }, provider: "openai" }, () => { calls++; return new Response("provider"); }); + assert.equal(result.status, 503); + assert.equal(calls, 0); +}); + +function database() { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec(readFileSync(new URL("../intelligence-migrations/0033_tenant_hierarchy.sql", import.meta.url), "utf8")); + sqlite.exec("CREATE TABLE control_users (username TEXT PRIMARY KEY)"); + sqlite.exec("INSERT INTO control_users VALUES ('alice'),('bob'),('operator')"); + const db = { + prepare(sql) { + let values = []; + return { bind(...params) { values = params; return this; }, + async all() { return { results: sqlite.prepare(sql).all(...values) }; }, + async first() { return sqlite.prepare(sql).get(...values) ?? null; }, + async run() { return { meta: { changes: Number(sqlite.prepare(sql).run(...values).changes) } }; } }; + }, + async batch(statements) { + sqlite.exec("BEGIN IMMEDIATE"); + try { const results = []; for (const statement of statements) results.push(await statement.run()); sqlite.exec("COMMIT"); return results; } + catch (error) { sqlite.exec("ROLLBACK"); throw error; } + }, + }; + const env = { ORGANISATIONS_ENABLED: "true", INTELLIGENCE_DB: db }; + const call = async (operation, values = {}) => { + const response = await handleManagedStateRequest(new Request("http://intelligence.internal/v1/managed-state/tenants", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, operation, ...values }) }), env); + return { status: response.status, body: await response.json() }; + }; + return { sqlite, db, env, call, close() { sqlite.close(); } }; +} +async function hierarchy(f) { + const org = (await f.call("org_create", { actor: "operator", data: { name: "Example" } })).body.result; + const team = (await f.call("team_create", { actor: "operator", org_id: org.id, data: { name: "Support" } })).body.result; + return { org, team }; +} +async function member(f, org, team, data = {}) { + return f.call("member_set", { actor: "operator", org_id: org.id, principal: "alice", revision: 0, + data: { role: "member", team_id: team?.id ?? null, status: "active", ...data } }); +} + +test("D1 hierarchy has explicit bindings, matching namespaces and content-free audits", async () => { + const f = database(); + try { + const { org, team } = await hierarchy(f); + assert.equal((await member(f, org, team)).status, 200); + assert.equal(tenantNamespace(await resolveAccountTenant(f.db, "alice", f.env)), ""); + assert.deepEqual((await f.call("workspaces", { principal: "bob" })).body.result.memberships, []); + const selected = await f.call("binding_set", { actor: "alice", principal: "alice", revision: 0, data: { org_id: org.id, team_id: team.id } }); + assert.equal(selected.status, 200); + const context = await resolveAccountTenant(f.db, "alice", f.env); + assert.equal(tenantNamespace(context), `org:${org.id}/team:${team.id}`); + assert.equal(context.grants_revision, 4); + const copied = await f.call("binding_set", { actor: "alice", principal: "bob", revision: 0, data: { org_id: org.id, team_id: team.id } }); + assert.equal(copied.status, 403); + const audit = f.sqlite.prepare("SELECT * FROM tenant_audit").all(); + assert.equal(audit.length, 4); + assert.equal(JSON.stringify(audit).includes("Support"), false); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_bindings").get().n, 1); + } finally { f.close(); } +}); + +test("D1 atomic membership plus binding and stale updates retain both old rows", async () => { + const f = database(); + try { + const { org, team } = await hierarchy(f); + assert.equal((await member(f, org, team, { bind: true, binding_revision: 0 })).status, 200); + const result = await f.call("member_set", { actor: "operator", org_id: org.id, principal: "alice", revision: 1, + data: { role: "billing", bind: true, binding_revision: 0 } }); + assert.equal(result.status, 412); + assert.equal(f.sqlite.prepare("SELECT role FROM tenant_memberships").get().role, "member"); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_audit").get().n, 4); + const missing = await f.call("org_update", { actor: "operator", org_id: org.id, data: { name: "Other" } }); + assert.equal(missing.status, 428); + assert.equal((await f.call("org_update", { actor: "operator", org_id: org.id, revision: 0, data: { name: "Other" } })).status, 412); + } finally { f.close(); } +}); + +test("D1 revision changes are fresh and foreign, revoked and missing workspaces are refused", async () => { + const f = database(); + try { + const { org, team } = await hierarchy(f), foreign = await hierarchy(f); + assert.equal((await member(f, org, foreign.team)).status, 404); + await member(f, org, team, { bind: true, binding_revision: 0 }); + const before = await resolveAccountTenant(f.db, "alice", f.env); + await f.call("member_set", { actor: "operator", org_id: org.id, principal: "alice", revision: 1, data: { role: "billing" } }); + assert.ok((await resolveAccountTenant(f.db, "alice", f.env)).grants_revision > before.grants_revision); + await f.call("team_update", { actor: "operator", org_id: org.id, team_id: team.id, revision: 1, data: { status: "deactivated" } }); + await assert.rejects(resolveAccountTenant(f.db, "alice", f.env), error => error.code === "workspace_forbidden" && error.status === 403); + f.sqlite.prepare("UPDATE tenant_bindings SET team_id='absent'").run(); + await assert.rejects(resolveAccountTenant(f.db, "alice", f.env), error => error.code === "workspace_not_found" && error.status === 404); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_teams").get().n, 2); + } finally { f.close(); } +}); + +test("D1 caps include deactivated rows and remain inside conditional writes", async () => { + const f = database(); + try { + const { org } = await hierarchy(f); + for (let i = 0; i < 99; i++) f.sqlite.prepare("INSERT INTO tenant_teams VALUES (?,?,?,'deactivated',1)").run(`team_${i}`, org.id, "Example"); + assert.equal((await f.call("team_create", { actor: "operator", org_id: org.id, data: { name: "Extra" } })).body.error.code, "tenant_limit_reached"); + for (let i = 0; i < 1000; i++) f.sqlite.prepare("INSERT INTO tenant_memberships VALUES (?,?,NULL,'member','deactivated',1)").run(org.id, `user_${i}`); + const capped = await member(f, org, null); + assert.equal(capped.status, 409); + assert.equal(capped.body.error.code, "tenant_limit_reached"); + } finally { f.close(); } +}); + +test("D1 CAS blocks a lost update and audit failure rolls the mutation back", async () => { + const f = database(); + try { + const { org } = await hierarchy(f); + // Interleave a competing write after the read but before the guarded atomic batch. + const original = f.db.batch; + f.db.batch = async statements => { + f.sqlite.prepare("UPDATE tenant_organisations SET revision=revision+1 WHERE id=?").run(org.id); + return original(statements); + }; + const conflict = await f.call("org_update", { actor: "operator", org_id: org.id, revision: 1, data: { name: "confidential-change" } }); + assert.equal(conflict.status, 412); + assert.equal(f.sqlite.prepare("SELECT name FROM tenant_organisations").get().name, "Example"); + f.db.batch = original; + f.sqlite.exec("CREATE TRIGGER reject_audit BEFORE INSERT ON tenant_audit BEGIN SELECT RAISE(ABORT,'audit unavailable'); END"); + const failed = await f.call("org_update", { actor: "operator", org_id: org.id, revision: 2, data: { name: "confidential-change" } }); + assert.equal(failed.status, 503); + assert.equal(f.sqlite.prepare("SELECT revision FROM tenant_organisations").get().revision, 2); + assert.equal(f.sqlite.prepare("SELECT name FROM tenant_organisations").get().name, "Example"); + } finally { f.close(); } +}); + +test("native requests ignore headers and body scope and resolve before dispatch", async () => { + const f = database(); + try { + const { org, team } = await hierarchy(f); + await member(f, org, team, { bind: true, binding_revision: 0 }); + const contexts = []; + const request = () => new Request("https://gateway.example/v1/chat/completions", { method: "POST", headers: { "X-Org": "foreign", "X-Team": "foreign", "X-MultiLLM-Workspace": "foreign" }, + body: JSON.stringify({ model: "test", org_id: "foreign", team_id: "foreign" }) }); + const authority = { authenticated: true, principal: { id: "alice" }, provider: "openai" }; + const result = await nativeGenerationFetch(request(), f.env, {}, authority, async (req, env, resolved) => { + contexts.push(resolved.tenantContext); return new Response("unchanged"); + }); + assert.equal(await result.text(), "unchanged"); + assert.equal(contexts[0].org_id, org.id); + assert.equal(contexts[0].team_id, team.id); + await f.call("org_update", { actor: "operator", org_id: org.id, revision: 1, data: { status: "deactivated" } }); + const rejected = await nativeGenerationFetch(request(), f.env, {}, authority, () => { throw Error("No provider"); }); + assert.equal(rejected.status, 403); + assert.equal((await rejected.json()).error.code, "workspace_forbidden"); + } finally { f.close(); } +}); + +test("prefix lookup leaves tenant resolution to the verified principal", async () => { + const row = { username: "alice" }; + const db = { prepare(sql) { + assert.ok(sql.includes("api_key_prefix=?")); + return { bind() { return this; }, async all() { return { results: [row] }; } }; + } }; + assert.deepEqual(await activeUsersByPrefix(db, "fixture", { ORGANISATIONS_ENABLED: "true" }), [row]); + assert.equal(Object.hasOwn(row, "tenant_context"), false); +}); + +const accountKey = "shared01-synthetic-matching"; +const otherKey = "shared01-synthetic-other"; +const integrationKey = "mllm_intelligence_" + "s".repeat(32); +const hash = key => `scrypt:32768:8:1$syntheticsalt$${scryptSync(key, "syntheticsalt", 64, + { N: 32768, r: 8, p: 1, maxmem: 64 * 1024 * 1024 }).toString("hex")}`; +const accountRow = (username, key) => ({ ...Object.fromEntries(USER_FIELDS.map(name => [name, null])), + username, api_key_hash: hash(key), api_key_prefix: `mllm_${key.slice(0, 8)}`, + scopes: "knowledge:read,chat,audio", is_admin: 0, created_at: "2026-10-09T00:00:00Z" }); +const rows = [accountRow("alice", otherKey), accountRow("bob", accountKey)]; +const integrationRow = { id: "integration:synthetic", scopes: JSON.stringify(["knowledge:read", "chat", "audio"]), + version: 1, created_at: "2026-10-09T00:00:00Z", revoked_at: null, + key_prefix: integrationKey.slice(0, "mllm_intelligence_".length + 16), key_hash: hash(integrationKey) }; +function authFixture() { + const f = database(), calls = { tenants: [], writes: 0, dispatches: 0 }; + const db = { prepare(sql) { + if (sql.includes("tenant_")) { + calls.tenants.push(sql); + const stmt = f.db.prepare(sql), bind = stmt.bind; + stmt.bind = (...values) => { if (sql.includes("WHERE b.principal=?")) calls.principal = values[0]; return bind.apply(stmt, values); }; + return stmt; + } + return { bind() { return this; }, async all() { return { results: sql.includes("control_users") ? rows : [] }; }, + async first() { return sql.includes("intelligence_credentials") ? integrationRow : null; }, + async run() { calls.writes++; return { meta: { changes: 1 } }; } }; + } }; + const env = { ...f.env, INTELLIGENCE_DB: db, AUTH_STORAGE_BACKEND: "d1", ADMIN_API_KEY: "synthetic-bootstrap", + ADMIN_USERNAME: "operator", JWT_SECRET: "synthetic-signing", REALTIME_ENABLED: "true", + REALTIME_PROVIDERS_JSON: '{"openai:realtime-test":{"url":"wss://approved.example/realtime"}}', + KNOWLEDGE_SERVICE: { async fetch() { calls.dispatches++; return Response.json({ version: 1, result: { status: "ok" } }); } } }; + return { ...f, env, calls }; +} +const knowledgeRequest = (key, mcp = false) => new Request(mcp ? "https://gateway.example/mcp" : "https://gateway.example/v1/knowledge/artifacts/synthetic", + { method: mcp ? "POST" : "GET", headers: { authorization: `Bearer ${key}`, "content-type": "application/json", "X-Org": "spoofed" }, + ...(mcp ? { body: JSON.stringify({ jsonrpc: "2.0", id: 7, method: "ping" }) } : {}) }); +const realtimeRequest = key => new Request("https://gateway.example/v1/realtime/client_secrets", { + method: "POST", headers: { authorization: `Bearer ${key}`, "content-type": "application/json", "X-Team": "spoofed" }, + body: JSON.stringify({ model: "openai:realtime-test" }) }); +const authRoutes = [ + ["knowledge", (f, key) => handleKnowledgeEdgeRequest(knowledgeRequest(key), f.env)], + ["realtime", (f, key) => handleRealtimeRequest(realtimeRequest(key), f.env)], +]; +for (const [route, send] of authRoutes) { + test(`${route} resolves only the matching account despite a forbidden prefix candidate`, async () => { + const f = authFixture(); + try { + const { org } = await hierarchy(f); + await member(f, org, null, { bind: true, binding_revision: 0 }); + await f.call("org_update", { actor: "operator", org_id: org.id, revision: 1, data: { status: "deactivated" } }); + assert.equal((await send(f, accountKey)).status, 200); + assert.equal(f.calls.principal, "bob"); + } finally { f.close(); } + }); + for (const kind of ["account", "bootstrap", "integration"]) { + test(`${route} classifies workspace failures for verified ${kind} credentials`, async () => { + const f = authFixture(), key = kind === "account" ? otherKey : kind === "bootstrap" ? f.env.ADMIN_API_KEY : integrationKey; + const principal = kind === "account" ? "alice" : kind === "bootstrap" ? "operator" : integrationRow.id; + try { + const { org } = await hierarchy(f); + await f.call("member_set", { actor: "operator", org_id: org.id, principal: "alice", revision: 0, + data: { role: "member", bind: true, binding_revision: 0 } }); + f.sqlite.prepare("UPDATE tenant_memberships SET principal=?").run(principal); + f.sqlite.prepare("UPDATE tenant_bindings SET principal=?").run(principal); + await f.call("org_update", { actor: "operator", org_id: org.id, revision: 1, data: { status: "deactivated" } }); + for (const [status, code, mutate] of [ + [403, "workspace_forbidden", () => {}], + [404, "workspace_not_found", () => f.sqlite.prepare("UPDATE tenant_bindings SET org_id='absent'").run()], + [503, "tenant_storage_unavailable", () => f.sqlite.exec("DROP TABLE tenant_bindings")], + ]) { + mutate(); const response = await send(f, key); + assert.equal(response.status, status); assert.equal((await response.json()).error.code, code); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.equal(f.calls.writes, 0); assert.equal(f.calls.dispatches, 0); + } + } finally { f.close(); } + }); + } + test(`${route} with organisations off never queries tenants or changes responses`, async () => { + const f = authFixture(); + try { + for (const flag of [undefined, "", "false", "malformed"]) { + f.env.ORGANISATIONS_ENABLED = flag; + for (const key of [accountKey, f.env.ADMIN_API_KEY, integrationKey]) assert.equal((await send(f, key)).status, 200); + } + assert.deepEqual(f.calls.tenants, []); + } finally { f.close(); } + }); + test(`${route} invalid keys never resolve a tenant`, async () => { + const f = authFixture(); + try { + assert.equal((await send(f, "shared01-wrong-key")).status, 401); + assert.deepEqual(f.calls.tenants, []); assert.equal(f.calls.writes, 0); + } finally { f.close(); } + }); +} + +test("Knowledge MCP preserves its JSON-RPC error envelope for forbidden workspaces", async () => { + const f = authFixture(); + try { + const { org } = await hierarchy(f); + await member(f, org, null, { bind: true, binding_revision: 0 }); + await f.call("org_update", { actor: "operator", org_id: org.id, revision: 1, data: { status: "deactivated" } }); + const response = await handleKnowledgeEdgeRequest(knowledgeRequest(otherKey, true), f.env); + assert.equal(response.status, 403); + const body = await response.json(); + assert.equal(body.jsonrpc, "2.0"); assert.equal(body.error.code, -32000); + assert.equal(body.error.message, "workspace_forbidden"); assert.equal(f.calls.dispatches, 0); + } finally { f.close(); } +}); + +test("D1 rechecks team cap inside the write when another team wins the last slot", async () => { + const f = database(); + try { + const { org } = await hierarchy(f); + for (let i = 0; i < 98; i++) f.sqlite.prepare("INSERT INTO tenant_teams VALUES (?,?,?,'active',1)").run(`team_${i}`, org.id, "Example"); + const original = f.db.batch; + f.db.batch = async statements => { + f.sqlite.prepare("INSERT INTO tenant_teams VALUES (?,?,?,'active',1)").run("last_slot", org.id, "Other"); + return original(statements); + }; + const response = await f.call("team_create", { actor: "operator", org_id: org.id, data: { name: "Extra" } }); + assert.equal(response.status, 409); + assert.equal(response.body.error.code, "tenant_limit_reached"); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_teams").get().n, 100); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_audit").get().n, 2); + } finally { f.close(); } +}); + +test("D1 malformed fields and unlisted operations never produce state changes", async () => { + const f = database(); + try { + const { org } = await hierarchy(f); + for (const data of [{ name: null }, { name: "Example", id: "caller-chosen" }, { name: "Example", status: null }, { name: "Example", parent_id: org.id }]) { + assert.equal((await f.call("org_create", { actor: "operator", data })).status, 400); + } + for (const data of [{ role: null }, { role: ["admin"] }, { status: null }, { status: ["active"] }]) { + assert.equal((await member(f, org, null, data)).status, 400); + } + assert.equal((await f.call("unknown", { statement: "DROP TABLE tenant_organisations" })).status, 400); + assert.equal(f.sqlite.prepare("SELECT count(*) AS n FROM tenant_memberships").get().n, 0); + } finally { f.close(); } +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..157d9ddc 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,7 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "ORGANISATIONS_ENABLED", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..79880fbf 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "tenant_organisations", "tenant_teams", "tenant_memberships", "tenant_bindings", "tenant_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/gateway-extensions.mjs b/worker/gateway-extensions.mjs index 3eed5a3f..59a47807 100644 --- a/worker/gateway-extensions.mjs +++ b/worker/gateway-extensions.mjs @@ -1,4 +1,5 @@ /** Static registration boundary for native dispatch; later integrations supply named hooks. */ +import { organisationsEnabled, resolveAccountTenant, tenantNamespace, TenantError } from "./tenants-d1.mjs"; import { generationCacheSettings, prepareNativeCache, cacheServedEvent } from "./exact-generation-cache.mjs"; import { createGatewayLifecycle } from "./gateway-lifecycle.mjs"; import { observeNativeResponse, appendNativeMetrics, PARSER_LIMIT } from "./request-telemetry.mjs"; @@ -72,6 +73,7 @@ async function nativeContext(request, env, authority, metrics) { .map(([name, state]) => [name, state.revision])) : {}; return Object.freeze({ provider: authority.provider, principal, model: typeof model === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,223}$/.test(model) ? model : null, + ...(organisationsEnabled(env) ? { tenantContext: authority.tenantContext, tenantNamespace: tenantNamespace(authority.tenantContext) } : {}), modelGroup: admissionModelGroup(authority.provider, model), requestId: crypto.randomUUID(), endpoint: authority.route, startedAt, retentionPolicy, cacheRevisions }); } @@ -322,6 +324,16 @@ async function runNativeLifecycle(request, env, ctx, authority, fetcher, registr } export async function nativeGenerationFetch(request, env, ctx, authority, fetcher, collaborators = []) { + if (organisationsEnabled(env) && authority.authenticated === true) { + try { + const tenantContext = await resolveAccountTenant(env.INTELLIGENCE_DB, authority.principal?.id ?? authority.keyId, env); + authority = { ...authority, tenantContext, tenantNamespace: tenantNamespace(tenantContext) }; + } catch (error) { + authority.deadlineHook?.deadline?.stop(); + if (error instanceof TenantError) return error.response(); + throw error; + } + } const path = authority.route ?? new URL(request.url).pathname; if (!generationPath(request, path)) return fetcher(request, env, authority); // The route has already authenticated the bootstrap key. Only public route metadata is trusted. diff --git a/worker/knowledge-edge.mjs b/worker/knowledge-edge.mjs index cb4a6a24..8ea18665 100644 --- a/worker/knowledge-edge.mjs +++ b/worker/knowledge-edge.mjs @@ -8,6 +8,7 @@ import { Buffer } from "node:buffer"; import { createHash, scrypt, timingSafeEqual } from "node:crypto"; import catalogue from "./knowledge-mcp-catalogue.json" with { type: "json" }; +import { organisationsEnabled, resolveAccountTenant, TenantError } from "./tenants-d1.mjs"; import { authStorageBackend } from "./container-env.mjs"; import { activeUsersByPrefix, adminUsernames, grantsAdmin, keyControlsPermit, validUser } from "./control-users-d1.mjs"; import { INTEGRATION_SCOPES, lookupIntegrationPrincipal } from "./intelligence-auth-d1.mjs"; @@ -158,6 +159,19 @@ function accountScopes(user, env) { return user.scopes.split(",").map(scope => scope.trim()).filter(scope => KNOWLEDGE_SCOPES.includes(scope)); } +async function tenantPrincipal(env, principal) { + if (!organisationsEnabled(env)) return { principal }; + try { + Object.defineProperty(principal, "tenant_context", { + value: await resolveAccountTenant(env.INTELLIGENCE_DB, principal.id, env), enumerable: false, + }); + return { principal }; + } catch (error) { + if (error instanceof TenantError) throw new KnowledgeEdgeError(error.code, error.message, error.status); + throw error; + } +} + // Dashboard accounts in D1 use the Container's key prefix and Werkzeug hashes. async function accountPrincipal(env, key, clientAddress) { if (key.length > MAX_KEY_LENGTH) return { denied: true }; @@ -174,7 +188,7 @@ async function accountPrincipal(env, key, clientAddress) { // An expired key, or one used outside its address ranges, is never served here; the // Container refuses it with key_expired or ip_not_allowed. if (index >= 0 && !keyControlsPermit(verifiable[index], clientAddress)) return null; - if (index >= 0) return { principal: { id: verifiable[index].username, scopes: accountScopes(verifiable[index], env), secret_scan_mode: verifiable[index].secret_scan_mode } }; + if (index >= 0) return tenantPrincipal(env, { id: verifiable[index].username, scopes: accountScopes(verifiable[index], env), secret_scan_mode: verifiable[index].secret_scan_mode }); // A hash format the edge cannot check is left to the Container. return verifiable.length < users.length ? null : { denied: true }; } @@ -188,7 +202,7 @@ async function resolvePrincipal(request, env) { if (!key) return null; if (env.ADMIN_API_KEY && sameSecret(key, env.ADMIN_API_KEY)) { const id = adminUsername(env); - return id ? { principal: { id, scopes: [...KNOWLEDGE_SCOPES] } } : null; + return id ? tenantPrincipal(env, { id, scopes: [...KNOWLEDGE_SCOPES] }) : null; } if (!env.INTELLIGENCE_DB) return null; if (!key.startsWith(KEY_NAMESPACE)) { @@ -205,7 +219,7 @@ async function resolvePrincipal(request, env) { if (record === null) return { denied: true }; if (!validRecord(record, prefix)) return null; if (record.revokedAt !== null || await matchingHash(key, [record.keyHash], prefix) < 0) return { denied: true }; - return { principal: { id: record.id, scopes: record.scopes.filter(scope => KNOWLEDGE_SCOPES.includes(scope)) } }; + return tenantPrincipal(env, { id: record.id, scopes: record.scopes.filter(scope => KNOWLEDGE_SCOPES.includes(scope)) }); } const permits = (principal, scope) => principal.scopes.includes(scope); diff --git a/worker/managed-state-dispatch.mjs b/worker/managed-state-dispatch.mjs index 6c6da481..66f6ee16 100644 --- a/worker/managed-state-dispatch.mjs +++ b/worker/managed-state-dispatch.mjs @@ -1,4 +1,5 @@ /** Fixed private managed-state domains; sibling authorities are injected explicitly. */ +import { handleTenantRequest, TENANT_STORE_PATH } from "./tenants-d1.mjs"; import { handleIdempotencyRequest } from "./idempotency-d1.mjs"; import { handleResponsesStateRequest } from "./responses-state-d1.mjs"; import { handleContextPageStateRequest } from "./context-pages-d1.mjs"; @@ -6,6 +7,7 @@ import { handleUsageReceiptStoreRequest } from "./usage-receipts.mjs"; export function handleManagedStateRequest(request, env, { reservations, toolGrants } = {}) { const url = new URL(request.url); + if (url.pathname === TENANT_STORE_PATH) return handleTenantRequest(request, env); if (!url.pathname.startsWith("/v1/managed-state/")) return null; if (url.origin !== "http://intelligence.internal" || url.search || url.hash || url.username || url.password || request.method !== "POST") { return Response.json({ version: 1, error: { code: "invalid_store_target", message: "Invalid managed storage target." } }, { status: 400 }); diff --git a/worker/realtime.mjs b/worker/realtime.mjs index 2c3fe957..e1fa50ba 100644 --- a/worker/realtime.mjs +++ b/worker/realtime.mjs @@ -2,6 +2,7 @@ import { Buffer } from "node:buffer"; import { createHash, scrypt, timingSafeEqual } from "node:crypto"; import { activeUsersByPrefix, boundedBody, keyControlsPermit, validUser } from "./control-users-d1.mjs"; +import { organisationsEnabled, resolveAccountTenant, TenantError } from "./tenants-d1.mjs"; import { lookupIntegrationPrincipal } from "./intelligence-auth-d1.mjs"; import { authStorageBackend } from "./container-env.mjs"; import { acquireAdmission, admissionSettings, principalHash } from "./admission-do.mjs"; @@ -92,10 +93,22 @@ function integration(row) { return { owner: row.id, scopes: row.scopes, revoked_at: row.revokedAt, credential_kind: "integration", credential_fingerprint: digest(row.keyHash) }; } +async function tenantPrincipal(env, principal) { + if (!organisationsEnabled(env)) return principal; + try { + Object.defineProperty(principal, "tenant_context", { + value: await resolveAccountTenant(env.INTELLIGENCE_DB, principal.owner, env), enumerable: false, + }); + return principal; + } catch (error) { + if (error instanceof TenantError) throw new RealtimeError(error.code, error.status); + throw error; + } +} async function authenticate(request, env, model, now) { const key = bearer(request); if (!key || key.length > 1024) throw new RealtimeError("invalid_api_key", 401); - if (key.startsWith("rt1.")) return consumeTicket(key, request, env, model, now); + if (key.startsWith("rt1.")) return tenantPrincipal(env, await consumeTicket(key, request, env, model, now)); let principal; if (same(key, env.ADMIN_API_KEY)) principal = bootstrap(env); else if (/^mllm_intelligence_[A-Za-z0-9_-]{32,128}$/.test(key)) { @@ -109,6 +122,7 @@ async function authenticate(request, env, model, now) { if (await checkHash(key, row.api_key_hash)) { principal = account(row); break; } } } if (!principal) throw new RealtimeError("invalid_api_key", 401); + await tenantPrincipal(env, principal); principal.principal_hash = await principalHash(`realtime-key:${digest(key)}`); return permit(principal, request, model, now); } diff --git a/worker/tenants-d1.mjs b/worker/tenants-d1.mjs new file mode 100644 index 00000000..6c310fd7 --- /dev/null +++ b/worker/tenants-d1.mjs @@ -0,0 +1,262 @@ +/** Fixed private tenant operations; no caller-selected identity or executable SQL. */ +import { createHash } from "node:crypto"; +import { AuthorityOperation, TenantContext, legacyTenant } from "./enterprise-contract.mjs"; + +export const TENANT_STORE_PATH = "/v1/managed-state/tenants"; +const MAX_BYTES = 8192; +const OBJECT = value => value !== null && typeof value === "object" && !Array.isArray(value); +const IDENTIFIER = /^[A-Za-z0-9_:.\-]{1,128}$/; +const text = (value, maximum = 128) => typeof value === "string" && value.trim().length > 0 && value.length <= maximum && !/[\x00-\x1f\x7f]/.test(value); +const id = () => crypto.randomUUID().replaceAll("-", ""); +let warned = false; +export function organisationsEnabled(env = {}, warn = message => console.warn(message)) { + const flag = env.ORGANISATIONS_ENABLED === undefined ? "" : typeof env.ORGANISATIONS_ENABLED === "string" ? env.ORGANISATIONS_ENABLED.trim().toLowerCase() : "invalid"; + if (["", "false", "0", "no", "off"].includes(flag)) return false; + if (["true", "1", "yes", "on"].includes(flag)) return true; + if (!warned) { warned = true; warn("Invalid ORGANISATIONS_ENABLED; organisations disabled"); } + return false; +} +export class TenantError extends Error { + constructor(code, status = 400) { super(code); this.code = code; this.status = status; } + response(privateReply = false) { + return Response.json({ ...(privateReply ? { version: 1, status: this.status } : {}), + error: { code: this.code, message: "The workspace operation could not be authorized or stored." } }, + { status: this.status, headers: { "cache-control": "no-store" } }); + } +} +function identifier(value) { if (typeof value !== "string" || !IDENTIFIER.test(value)) throw new TenantError("invalid_request"); return value; } +function revision(value) { + if (value === undefined || value === null) throw new TenantError("revision_required", 428); + if (!Number.isSafeInteger(value) || value < 0 || value >= 9007199254740000) throw new TenantError("invalid_revision"); + return value; +} +function requireText(value, maximum) { if (!text(value, maximum)) throw new TenantError("invalid_request"); return value; } +function fields(value, allowed, required = []) { + if (!OBJECT(value) || Object.keys(value).some(name => !allowed.includes(name)) || required.some(name => !Object.hasOwn(value, name))) throw new TenantError("invalid_request"); +} +function cas(row, value) { if ((row?.revision ?? 0) !== revision(value)) throw new TenantError("revision_conflict", 412); } +function legacy(principal) { + const principal_id = IDENTIFIER.test(principal) ? principal : `principal:${createHash("sha256").update(principal).digest("hex")}`; + return legacyTenant(new AuthorityOperation({ context: new TenantContext({ principal_id }), scoped_id: "workspace", revision: 0, operation_id: "resolve" })); +} +export function tenantNamespace(context) { + return context.org_id === null ? "" : `org:${context.org_id}${context.team_id === null ? "" : `/team:${context.team_id}`}`; +} +const one = (db, sql, values = []) => db.prepare(sql).bind(...values).first(); +const rows = async (db, sql, values = []) => (await db.prepare(sql).bind(...values).all()).results; +async function requireSchema(db) { + if (!db) throw new TenantError("tenant_storage_unavailable", 503); + // Reads also verify the complete authority, so an absent binding table cannot mean legacy. + for (const sql of ["SELECT id,name,status,revision FROM tenant_organisations LIMIT 0", + "SELECT id,org_id,name,status,revision FROM tenant_teams LIMIT 0", + "SELECT org_id,principal,team_id,role,status,revision FROM tenant_memberships LIMIT 0", + "SELECT principal,org_id,team_id,revision FROM tenant_bindings LIMIT 0", + "SELECT id,actor,action,org_id,team_id,principal,old_revision,new_revision,at FROM tenant_audit LIMIT 0"]) { + await db.prepare(sql).all(); + } +} +async function organisation(db, orgId, active = false) { + const org = await one(db, "SELECT * FROM tenant_organisations WHERE id=?", [identifier(orgId)]); + if (!org) throw new TenantError("workspace_not_found", 404); + if (active && org.status !== "active") throw new TenantError("workspace_forbidden", 403); + return org; +} +async function team(db, orgId, teamId, active = false) { + const value = await one(db, "SELECT * FROM tenant_teams WHERE org_id=? AND id=?", [orgId, identifier(teamId)]); + if (!value) throw new TenantError("workspace_not_found", 404); + if (active && value.status !== "active") throw new TenantError("workspace_forbidden", 403); + return value; +} +async function workspace(db, principal, orgId, teamId) { + await organisation(db, orgId, true); + if (teamId !== null) await team(db, orgId, teamId, true); + const membership = await one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", [orgId, principal]); + if (!membership || membership.status !== "active" || membership.team_id !== teamId) throw new TenantError("workspace_forbidden", 403); +} +async function resolve(db, principal) { + const value = await one(db, `SELECT b.org_id,b.team_id,b.revision AS binding_revision, + o.id AS found_org,o.status AS org_status,o.revision AS org_revision, + t.id AS found_team,t.status AS team_status,t.revision AS team_revision, + m.status AS member_status,m.team_id AS member_team,m.revision AS member_revision + FROM tenant_bindings b LEFT JOIN tenant_organisations o ON o.id=b.org_id + LEFT JOIN tenant_teams t ON t.id=b.team_id AND t.org_id=b.org_id + LEFT JOIN tenant_memberships m ON m.org_id=b.org_id AND m.principal=b.principal WHERE b.principal=?`, [principal]); + if (!value) return legacy(principal); + if (!value.found_org || value.team_id !== null && !value.found_team) throw new TenantError("workspace_not_found", 404); + if (value.org_status !== "active" || value.team_id !== null && value.team_status !== "active" + || value.member_status !== "active" || value.member_team !== value.team_id) throw new TenantError("workspace_forbidden", 403); + return new TenantContext({ principal_id: legacy(principal).principal_id, org_id: value.org_id, team_id: value.team_id, + grants_revision: value.binding_revision + value.org_revision + (value.team_revision ?? 0) + value.member_revision }); +} +export async function resolveAccountTenant(db, principal, env = {}) { + if (!organisationsEnabled(env)) return legacy(principal); + try { requireText(principal); await requireSchema(db); return await resolve(db, principal); } + catch (error) { if (error instanceof TenantError) throw error; throw new TenantError("tenant_storage_unavailable", 503); } +} +function audit(db, mutationId, v, action, orgId, teamId, principal, old, next) { + return db.prepare(`INSERT INTO tenant_audit (id,actor,action,org_id,team_id,principal,old_revision,new_revision,at) + SELECT ?,?,?,?,?,?,?,?,? WHERE changes()=1`).bind(mutationId, requireText(v.actor), action, orgId, teamId, principal, old, next, new Date().toISOString()); +} +async function mutate(db, statement, v, action, orgId, teamId, principal, old, next, extra = []) { + const mutationId = id(); + const statements = [statement, audit(db, mutationId, v, action, orgId, teamId, principal, old, next), ...extra.map(factory => factory(mutationId))]; + const result = await db.batch(statements); + if (result[0].meta.changes !== 1) throw new TenantError("revision_conflict", 412); +} +const ACTIVE_ORG = "EXISTS (SELECT 1 FROM tenant_organisations WHERE id=? AND status='active')"; +const ACTIVE_TEAM = "(? IS NULL OR EXISTS (SELECT 1 FROM tenant_teams WHERE id=? AND org_id=? AND status='active'))"; +const ACTIVE_MEMBER = "EXISTS (SELECT 1 FROM tenant_memberships WHERE org_id=? AND principal=? AND status='active' AND team_id IS ?)"; +const BINDING_CAS = "COALESCE((SELECT revision FROM tenant_bindings WHERE principal=?),0)=?"; + +async function namedWrite(db, operation, v) { + const creating = operation.endsWith("create"), isTeam = operation.startsWith("team"); + fields(v.data, ["name", "status"], creating ? ["name"] : []); + if (!Object.keys(v.data).length || creating && Object.hasOwn(v.data, "status") && v.data.status !== "active") throw new TenantError("invalid_request"); + const org = operation === "org_create" ? null : await organisation(db, v.org_id, creating && isTeam); + const previous = creating ? null : isTeam ? await team(db, v.org_id, v.team_id) : org; + if (!creating) cas(previous, v.revision); + const name = requireText(Object.hasOwn(v.data, "name") ? v.data.name : previous?.name, 200); + const status = Object.hasOwn(v.data, "status") ? v.data.status : previous?.status ?? "active"; + if (!["active", "deactivated"].includes(status)) throw new TenantError("invalid_request"); + const old = previous?.revision ?? 0, next = old + 1, target = creating ? id() : previous.id; + let statement; + if (isTeam && creating) { + if ((await one(db, "SELECT count(*) AS count FROM tenant_teams WHERE org_id=?", [v.org_id])).count >= 100) throw new TenantError("tenant_limit_reached", 409); + statement = db.prepare(`INSERT INTO tenant_teams SELECT ?,?,?,?,? WHERE ${ACTIVE_ORG} + AND (SELECT count(*) FROM tenant_teams WHERE org_id=?)<100`).bind(target, v.org_id, name, status, next, v.org_id, v.org_id); + } else if (creating) statement = db.prepare("INSERT INTO tenant_organisations VALUES (?,?,?,?)").bind(target, name, status, next); + else if (isTeam) statement = db.prepare("UPDATE tenant_teams SET name=?,status=?,revision=? WHERE id=? AND org_id=? AND revision=?").bind(name, status, next, target, v.org_id, old); + else statement = db.prepare("UPDATE tenant_organisations SET name=?,status=?,revision=? WHERE id=? AND revision=?").bind(name, status, next, target, old); + try { await mutate(db, statement, v, operation, isTeam ? v.org_id : target, isTeam ? target : null, null, old, next); } + catch (error) { + if (isTeam && creating && error instanceof TenantError) { + await organisation(db, v.org_id, true); + if ((await one(db, "SELECT count(*) AS count FROM tenant_teams WHERE org_id=?", [v.org_id])).count >= 100) throw new TenantError("tenant_limit_reached", 409); + } + throw error; + } + return { id: target, ...(isTeam ? { org_id: v.org_id } : {}), name, status, revision: next }; +} +async function memberWrite(db, v) { + const orgId = identifier(v.org_id), principal = requireText(v.principal); + fields(v.data, ["role", "team_id", "status", "bind", "binding_revision"]); + if (!Object.keys(v.data).length || "bind" in v.data && typeof v.data.bind !== "boolean" || "binding_revision" in v.data && !v.data.bind) throw new TenantError("invalid_request"); + await organisation(db, orgId); + if (!await one(db, "SELECT username FROM control_users WHERE username=?", [principal])) throw new TenantError("principal_not_found", 404); + const previous = await one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", [orgId, principal]); + cas(previous, v.revision); + if (!previous && (await one(db, "SELECT count(*) AS count FROM tenant_memberships WHERE org_id=?", [orgId])).count >= 1000) throw new TenantError("tenant_limit_reached", 409); + const role = Object.hasOwn(v.data, "role") ? v.data.role : previous?.role ?? "member"; + const status = Object.hasOwn(v.data, "status") ? v.data.status : previous?.status ?? "active"; + const teamId = Object.hasOwn(v.data, "team_id") ? v.data.team_id : previous?.team_id ?? null; + if (!["admin", "billing", "member"].includes(role) || !["active", "deactivated"].includes(status)) throw new TenantError("invalid_request"); + if (status === "active") await organisation(db, orgId, true); + if (teamId !== null) await team(db, orgId, teamId, status === "active"); + if (v.data.bind) { + if (status !== "active") throw new TenantError("workspace_forbidden", 403); + cas(await one(db, "SELECT * FROM tenant_bindings WHERE principal=?", [principal]), v.data.binding_revision); + } + const old = previous?.revision ?? 0, next = old + 1; + const conditions = `EXISTS (SELECT 1 FROM control_users WHERE username=?) AND + COALESCE((SELECT revision FROM tenant_memberships WHERE org_id=? AND principal=?),0)=? + AND (? > 0 OR (SELECT count(*) FROM tenant_memberships WHERE org_id=?)<1000) + AND (? <> 'active' OR (${ACTIVE_ORG} AND ${ACTIVE_TEAM})) + AND (? IS NULL OR EXISTS (SELECT 1 FROM tenant_teams WHERE id=? AND org_id=?)) + AND (?=0 OR ${BINDING_CAS})`; + const statement = db.prepare(`INSERT INTO tenant_memberships (org_id,principal,team_id,role,status,revision) + SELECT ?,?,?,?,?,? WHERE ${conditions} + ON CONFLICT(org_id,principal) DO UPDATE SET team_id=excluded.team_id,role=excluded.role,status=excluded.status,revision=excluded.revision`).bind( + orgId, principal, teamId, role, status, next, principal, orgId, principal, old, old, orgId, + status, orgId, teamId, teamId, orgId, teamId, teamId, orgId, v.data.bind ? 1 : 0, principal, v.data.binding_revision ?? 0); + const extra = v.data.bind ? [mutation => db.prepare(`INSERT INTO tenant_bindings (principal,org_id,team_id,revision) + SELECT ?,?,?,? WHERE EXISTS (SELECT 1 FROM tenant_audit WHERE id=?) + ON CONFLICT(principal) DO UPDATE SET org_id=excluded.org_id,team_id=excluded.team_id,revision=excluded.revision`) + .bind(principal, orgId, teamId, v.data.binding_revision + 1, mutation), + () => audit(db, id(), v, "binding_set", orgId, teamId, principal, v.data.binding_revision, v.data.binding_revision + 1)] : []; + try { await mutate(db, statement, v, "member_set", orgId, teamId, principal, old, next, extra); } + catch (error) { + if (error instanceof TenantError && error.code === "revision_conflict" && !previous + && (await one(db, "SELECT count(*) AS count FROM tenant_memberships WHERE org_id=?", [orgId])).count >= 1000) throw new TenantError("tenant_limit_reached", 409); + throw error; + } + return { org_id: orgId, principal, team_id: teamId, role, status, revision: next }; +} +async function bindingWrite(db, v) { + const principal = requireText(v.principal); + if (v.actor !== principal) throw new TenantError("workspace_forbidden", 403); + fields(v.data, ["org_id", "team_id"], ["org_id", "team_id"]); + const orgId = identifier(v.data.org_id), teamId = v.data.team_id; + await workspace(db, principal, orgId, teamId); + const previous = await one(db, "SELECT * FROM tenant_bindings WHERE principal=?", [principal]); + cas(previous, v.revision); + const old = previous?.revision ?? 0, next = old + 1; + const statement = db.prepare(`INSERT INTO tenant_bindings (principal,org_id,team_id,revision) + SELECT ?,?,?,? WHERE ${BINDING_CAS} AND ${ACTIVE_ORG} AND ${ACTIVE_TEAM} AND ${ACTIVE_MEMBER} + ON CONFLICT(principal) DO UPDATE SET org_id=excluded.org_id,team_id=excluded.team_id,revision=excluded.revision`).bind( + principal, orgId, teamId, next, principal, old, orgId, teamId, teamId, orgId, orgId, principal, teamId); + try { await mutate(db, statement, v, "binding_set", orgId, teamId, principal, old, next); } + catch (error) { if (error instanceof TenantError) await workspace(db, principal, orgId, teamId); throw error; } + return { principal, org_id: orgId, team_id: teamId, revision: next }; +} +const OPERATIONS = Object.freeze({ + org_list: [], org_get: ["org_id"], org_create: ["actor", "data"], org_update: ["actor", "org_id", "revision", "data"], + team_list: ["org_id"], team_create: ["actor", "org_id", "data"], team_update: ["actor", "org_id", "team_id", "revision", "data"], + member_list: ["org_id"], member_set: ["actor", "org_id", "principal", "revision", "data"], + workspaces: ["principal"], binding_set: ["actor", "principal", "revision", "data"], resolve: ["principal"], +}); +async function execute(db, v) { + if (v.operation === "resolve") return resolve(db, requireText(v.principal)); + if (v.operation === "org_list") return { organisations: await rows(db, "SELECT * FROM tenant_organisations ORDER BY id") }; + if (v.operation === "workspaces") { + const principal = requireText(v.principal); + const memberships = await rows(db, `SELECT m.* FROM tenant_memberships m JOIN tenant_organisations o ON o.id=m.org_id + LEFT JOIN tenant_teams t ON t.id=m.team_id AND t.org_id=m.org_id + WHERE m.principal=? AND m.status='active' AND o.status='active' AND (m.team_id IS NULL OR t.status='active') ORDER BY m.org_id`, [principal]); + return { memberships, binding: await one(db, "SELECT * FROM tenant_bindings WHERE principal=?", [principal]) + ?? { principal, org_id: null, team_id: null, revision: 0 } }; + } + if (v.operation === "binding_set") return bindingWrite(db, v); + if (v.operation.endsWith("create") || v.operation.endsWith("update")) return namedWrite(db, v.operation, v); + const org = await organisation(db, v.org_id); + if (v.operation === "org_get") return org; + if (v.operation === "team_list") return { teams: await rows(db, "SELECT * FROM tenant_teams WHERE org_id=? ORDER BY id", [v.org_id]) }; + if (v.operation === "member_list") return { members: await rows(db, "SELECT * FROM tenant_memberships WHERE org_id=? ORDER BY principal", [v.org_id]) }; + return memberWrite(db, v); +} +async function boundedJson(request) { + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json" || !request.body) throw new TenantError("invalid_request"); + const length = request.headers.get("content-length"); + if (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) throw new TenantError("invalid_request"); + const reader = request.body.getReader(), chunks = []; let size = 0; + try { + while (true) { + const { value, done } = await reader.read(); if (done) break; + size += value.byteLength; + if (size > MAX_BYTES) { void reader.cancel().catch(() => {}); throw new TenantError("invalid_request"); } + chunks.push(value); + } + const data = new Uint8Array(size); let offset = 0; + for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.length; } + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(data)); + } catch { throw new TenantError("invalid_request"); } + finally { reader.releaseLock(); } +} +export async function handleTenantRequest(request, env) { + if (!organisationsEnabled(env)) return new TenantError("not_found", 404).response(true); + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== TENANT_STORE_PATH || url.search || url.hash || url.username || url.password || request.method !== "POST") return new TenantError("not_found", 404).response(true); + let v; + try { + v = await boundedJson(request); + if (!OBJECT(v) || v.version !== 1 || !Object.hasOwn(OPERATIONS, v.operation)) throw new TenantError("invalid_request"); + const required = OPERATIONS[v.operation]; + fields(v, ["version", "operation", ...required], ["version", "operation", ...required.filter(name => name !== "revision")]); + if (required.includes("revision")) revision(v.revision); + if (required.includes("actor")) requireText(v.actor); + await requireSchema(env.INTELLIGENCE_DB); + const result = await execute(env.INTELLIGENCE_DB, v); + return Response.json({ version: 1, result }, { headers: { "cache-control": "no-store" } }); + } catch (error) { + return (error instanceof TenantError ? error : new TenantError("tenant_storage_unavailable", 503)).response(true); + } +} From 0cfdf92eadd8550ec69d4d2e9f7d1c1b581c8843 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:49:33 +0530 Subject: [PATCH 7/9] test(d1): start the integration Worker with the production nodejs_compat flag --- tests/test_d1_integration.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_d1_integration.mjs b/tests/test_d1_integration.mjs index 16642b95..a011f622 100644 --- a/tests/test_d1_integration.mjs +++ b/tests/test_d1_integration.mjs @@ -24,7 +24,8 @@ const bundled = await build({ stdin: { resolveDir: process.cwd(), sourcefile: "p async function privateStore(t, skip = []) { const mf = new Miniflare(convertV4MiniflareOptions({ modules: true, script: bundled.outputFiles[0].text, - compatibilityDate: "2026-07-30", host: "127.0.0.1", port: 0, d1Databases: ["INTELLIGENCE_DB"] })); + compatibilityDate: "2026-07-30", compatibilityFlags: ["nodejs_compat"], host: "127.0.0.1", port: 0, + d1Databases: ["INTELLIGENCE_DB"] })); t.after(() => mf.dispose()); const db = await mf.getD1Database("INTELLIGENCE_DB"); await applyMigrations(db, { skip }); From 83e9193749f0dd040c4f7afca030e50b125babe4 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 14:21:51 +0530 Subject: [PATCH 8/9] feat(gateway): wire the Wave 7 enterprise features together --- README.md | 6 + app.py | 3 +- docs/credits-ledger.md | 56 +- docs/payment-billing.md | 33 +- docs/saml-federation.md | 4 +- docs/scim-provisioning.md | 3 +- docs/tenant-governance.md | 10 + docs/tenant-hierarchy.md | 11 +- package.json | 2 +- route_helpers.py | 4 +- routes/chat_cache.py | 4 + routes/core.py | 7 +- routes/enterprise_preview.py | 2 +- routes/gateway_batches.py | 16 +- routes/media_batches.py | 2 + routes/payments.py | 8 +- routes/saml_federation.py | 3 +- services/accounted_dispatch.py | 26 +- services/auth_service.py | 11 + services/budget_service.py | 32 +- services/context_pages.py | 9 +- services/control_state_d1.py | 3 +- services/credits_admission.py | 213 +++++++ services/credits_ledger.py | 62 +- services/credits_payments.py | 33 + services/gateway_extensions.py | 87 +++ services/hedged_requests.py | 33 +- services/idempotency_store.py | 2 + services/intelligence_d1_store.py | 9 + services/managed_dispatch.py | 13 +- services/managed_turn.py | 6 +- services/media_signing.py | 27 + services/payment_billing.py | 5 +- services/request_accounting.py | 130 +++- services/reservation_store.py | 18 +- services/responses_state.py | 7 +- services/saml_federation.py | 10 +- services/semantic_generation_cache.py | 2 + services/shared_generation_cache.py | 7 + services/tenant_governance.py | 66 +- services/tenant_hierarchy.py | 76 ++- tests/fixtures/governance_reconciliation.json | 82 +++ tests/fixtures/secret_firewall_coverage.json | 1 + tests/test_cascades.py | 4 +- tests/test_credits_ledger.py | 5 +- tests/test_credits_ledger_worker.mjs | 22 +- tests/test_image_quality.py | 11 +- tests/test_image_quality_parallel_r2.py | 3 +- tests/test_saml_federation.py | 17 +- tests/test_tenant_governance.py | 25 + tests/test_tenant_governance_worker.mjs | 22 + tests/test_tenant_hierarchy.py | 38 ++ tests/test_wave7_managed.py | 569 ++++++++++++++++++ tests/test_wave7_registrar.py | 490 +++++++++++++++ tests/test_wave7_worker.mjs | 483 +++++++++++++++ worker/container-env.mjs | 1 + worker/context-pages-d1.mjs | 9 +- worker/control-users-d1.mjs | 12 + worker/cors-policy.mjs | 4 +- worker/credits-admission.mjs | 117 ++++ worker/credits-d1.mjs | 47 +- worker/exact-generation-cache.mjs | 4 +- worker/gateway-extensions.mjs | 27 +- worker/idempotency-d1.mjs | 4 +- worker/intelligence-outbound.mjs | 2 + worker/knowledge-edge.mjs | 3 +- worker/managed-state-dispatch.mjs | 23 +- worker/reservations-d1.mjs | 103 +++- worker/responses-state-d1.mjs | 4 +- worker/scim-d1.mjs | 18 +- worker/semantic-generation-cache.mjs | 36 +- worker/tenant-governance-d1.mjs | 74 ++- worker/tenants-d1.mjs | 50 ++ worker/usage-ledger-d1.mjs | 3 +- 74 files changed, 3179 insertions(+), 195 deletions(-) create mode 100644 services/credits_admission.py create mode 100644 services/credits_payments.py create mode 100644 tests/fixtures/governance_reconciliation.json create mode 100644 tests/test_wave7_managed.py create mode 100644 tests/test_wave7_registrar.py create mode 100644 tests/test_wave7_worker.mjs create mode 100644 worker/credits-admission.mjs diff --git a/README.md b/README.md index 952e8a7a..42f10fe9 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,12 @@ store data name the D1 migration to apply before enabling them. - [Context-leak canary tokens](docs/context-canary.md): `CONTEXT_CANARY_MODE=log` or `block` with `CONTEXT_CANARY_POLICY_JSON` detects opted-in disclosure of a request-specific marker. It does not claim complete leak detection. - [Signed usage receipts](docs/usage-receipts.md): `USAGE_RECEIPTS_ENABLED` with a reviewed Ed25519 key writes a hash-chained receipt per settled usage event that its owner can fetch and verify, keeping unknown cost explicit (migration 0032). - [Enterprise integration contracts](docs/enterprise-contracts.md): typed identity, tenancy, quota and credit boundaries for later organisation, SSO and payment modules. `ENTERPRISE_PREVIEW_ENABLED` shows a read-only administrator preview; nothing else activates. +- [Organisations and teams](docs/tenant-hierarchy.md): `ORGANISATIONS_ENABLED` adds optional organisation and team membership with a workspace identity the server verifies, and keeps each workspace's caches, idempotency, hosted state and usage separate (migration 0033). +- [Tenant governance](docs/tenant-governance.md): `TENANT_GOVERNANCE_ENABLED` intersects organisation, team and key grants, reserves organisation and team budgets together with the key's, and attributes usage to the verified workspace (migration 0034). +- [SAML federation](docs/saml-federation.md): `SAML_ENABLED` with `SAML_BROKER_URL`, `SAML_TRUST_CONFIG_JSON` and `SAML_CALLBACK_URL` signs gateway accounts in to the dashboard through an explicitly trusted SAML broker, only for subjects an administrator has linked to an account (migration 0035). +- [SCIM provisioning](docs/scim-provisioning.md): `SCIM_ENABLED` with `SCIM_TRUST_CONFIG_JSON` provisions scoped users and organisation teams through SCIM 2.0; it needs D1 storage and security revision sync (migration 0036). +- [Gateway credits](docs/credits-ledger.md): `CREDITS_ENABLED` keeps an append-only ledger of credits, holds and settlements, separate from usage estimates. `CREDITS_ENFORCEMENT=funded` or `all` makes requests reserve and settle credits (migration 0037). +- [Top-ups](docs/payment-billing.md): `PAYMENTS_ENABLED` with `PAYMENT_PROCESSOR_CONFIG_JSON` and `PAYMENT_WEBHOOK_KEY_REF` creates Stripe Checkout top-ups and credits explicit USD payments only from verified, idempotent webhooks (migration 0038). ## Setup diff --git a/app.py b/app.py index b74df539..eb8fa03b 100644 --- a/app.py +++ b/app.py @@ -43,7 +43,7 @@ from services.cache_service import CacheService from services.image_relay_catalog import ImageRelayCatalogRefresh from services.provider_catalog_refresh import ProviderCatalogAutoRefresh -from services.gateway_extensions import gateway_callbacks, register_gateway_extensions +from services.gateway_extensions import gateway_callbacks, register_gateway_extensions, register_enterprise_features from services.metrics_service import MetricsService from services.proxy_service import ProxyService from services import usage_ledger @@ -149,6 +149,7 @@ def create_app() -> Flask: register_config_snapshot_routes(app) register_enterprise_preview_routes(app) register_tenant_routes(app, csrf=csrf) + register_enterprise_features(app, csrf) register_shadow_eval_routes(app, csrf, AuthService, MetricsService, ProxyService) register_knowledge_routes(app, csrf) register_gateway_mcp_routes(app, csrf) diff --git a/docs/credits-ledger.md b/docs/credits-ledger.md index 5ace9fb7..5a9f03f0 100644 --- a/docs/credits-ledger.md +++ b/docs/credits-ledger.md @@ -133,6 +133,20 @@ and receipt errors must never trigger a second charge. Local tests exercise SQLite and a SQLite-backed D1 contract fake; they do not validate deployed D1, live provider metering or an operator's tariff. +A billing member's top-up funds that member's own usage within the verified +workspace; it does not fund the whole organisation. + +Verified Stripe payments use the dedicated private `payment_append` operation. +The callback records a credit for a top-up, or a negative `adjust` for a refund or +dispute, with the reserved `processor:stripe` actor. Processor adjustments can +make balance negative after spending; new reservations remain denied. Ordinary +admin writes cannot use any `processor:` actor. The immutable payment identity +is replay-safe even after re-reading the current ledger revision. The callback +retries only 412 revision conflicts, with at most three attempts, preserving the payment +operation ID and amount. Its result advances the payment event revision, which +is separate from the ledger revision. A failed callback leaves pending payment +evidence for retry and never grants a second credit. + Shared integer and idempotency vectors: ```json @@ -152,6 +166,46 @@ Shared integer and idempotency vectors: {"request":{"owner":"overrun","kind":"commit","amount_microusd":150,"operation_id":"charge","revision":2,"scoped_id":"attempt","tariff_revision":1},"revision":3,"summary":{"balance_microusd":-50,"held_microusd":0,"available_microusd":-50}}, {"request":{"owner":"overrun","kind":"reserve","amount_microusd":1,"operation_id":"denied","revision":3,"scoped_id":"next","tariff_revision":1},"error":"credits_insufficient"}, {"request":{"owner":"overrun","kind":"credit","amount_microusd":100,"operation_id":"refill","revision":3},"revision":4,"summary":{"balance_microusd":50,"held_microusd":0,"available_microusd":50}}, - {"request":{"owner":"overrun","kind":"reserve","amount_microusd":1,"operation_id":"next","revision":4,"scoped_id":"next","tariff_revision":1},"revision":5,"summary":{"balance_microusd":50,"held_microusd":1,"available_microusd":49}} + {"request":{"owner":"overrun","kind":"reserve","amount_microusd":1,"operation_id":"next","revision":4,"scoped_id":"next","tariff_revision":1},"revision":5,"summary":{"balance_microusd":50,"held_microusd":1,"available_microusd":49}}, + {"request":{"owner":"processor-tests","kind":"credit","amount_microusd":100,"operation_id":"fund","revision":0},"revision":1}, + {"request":{"owner":"processor-tests","kind":"reserve","amount_microusd":100,"operation_id":"hold","revision":1,"scoped_id":"attempt","tariff_revision":1},"revision":2}, + {"request":{"owner":"processor-tests","kind":"commit","amount_microusd":100,"operation_id":"spend","revision":2,"scoped_id":"attempt","tariff_revision":1},"revision":3}, + {"request":{"owner":"processor-tests","kind":"adjust","amount_microusd":-100,"operation_id":"admin-spoof","revision":3,"actor":"processor:stripe","reason":"refund"},"error":"invalid_credits_request"}, + {"operation":"payment_append","request":{"owner":"processor-tests","event_kind":"refund","amount_microusd":100,"operation_id":"refund:checkout","revision":2,"scoped_id":"checkout","processor_id":"stripe"},"error":"credits_revision_mismatch"}, + {"operation":"payment_append","request":{"owner":"processor-tests","event_kind":"refund","amount_microusd":100,"operation_id":"refund:checkout","revision":3,"scoped_id":"checkout","processor_id":"stripe"},"revision":4,"summary":{"balance_microusd":-100,"held_microusd":0,"available_microusd":-100}}, + {"operation":"payment_append","request":{"owner":"processor-tests","event_kind":"refund","amount_microusd":100,"operation_id":"refund:checkout","revision":4,"scoped_id":"checkout","processor_id":"stripe"},"revision":4,"summary":{"balance_microusd":-100,"revision":4}} ] ``` + +## Request admission + +`CREDITS_ENABLED=true` enables the ledger without charging requests. +`CREDITS_ENFORCEMENT` defaults to `off`; `funded` charges owners whose ledger +revision is above zero, and `all` charges every authenticated owner, including +administrators. Credit owners before selecting `all`. Empty or invalid settings +are off, and enforcement has effect only while the ledger is enabled. + +The owner is the verified principal within the selected organisation and team. +A member's payment funds that member's usage in that workspace. Legacy owners +keep their existing format. Key and workspace budgets reserve first; credits +reserve last against the same priced estimate, rounded up to integer micro-USD. +The price revision is the first 12 hexadecimal digits of SHA-256 of the exact +`MODEL_PRICING_USD_PER_MILLION` configuration, interpreted as an integer. + +Flask managed requests and native Worker generations use this admission policy. +Insufficient credit returns JSON 402 `credits_insufficient`; an unpriced charged +request returns 503 `credits_unpriced`. Authority failures return 503 +`credits_unavailable`. Revision conflicts allow at most three total attempts, +rereading the revision with the same operation ID, scope and amount. An uncertain write is never retried. +Measured usage commits its actual cost. Unknown usage records a reconciliation +marker and retains its hold until an evidenced operator reconciliation. +Pre-dispatch cancellation releases the original estimate once. Each hedged or +batch attempt has a separate reservation; a cancelled hedge's credit estimate +is released. Gateway estimates and measured costs are not provider invoices. + +Native generations and semantic embeddings use separate per-attempt credit reservations; cache hits release the generation estimate. + +Credit and governance admission runs after the idempotency claim and admission, +at the same point as key-budget admission. A refusal dispatches no provider and +leaves no open hold; its claim settles like a key-budget refusal. A caller who +retries after a top-up must use a new `Idempotency-Key`. diff --git a/docs/payment-billing.md b/docs/payment-billing.md index 22356db9..f9684c60 100644 --- a/docs/payment-billing.md +++ b/docs/payment-billing.md @@ -31,7 +31,7 @@ Payment routes require a migrated durable store. The private D1 domain is `http://intelligence.internal/v1/managed-state/payments`, with fixed reserve, save, find, claim and finish operations. `D1PaymentStore` accepts a private transport; `SqlPaymentStore` accepts an existing migrated SQLite database. -Neither creates tables or falls back to memory. Missing tables, missing +The application selects D1 when configured, otherwise the existing `USAGE_DB_PATH` SQLite store. Neither creates tables or falls back to memory. Missing tables, missing columns, unavailable storage or unconnected storage return JSON 503 `payments_unavailable` before any processor request. Apply the additive payment migration through the normal migration process before enabling the flag. @@ -50,9 +50,10 @@ Send a gateway key to `POST /v1/payments/checkout` with: ``` The default permission grants only the verified key owner in the legacy -single-principal scope. Organisation workspaces require the injected permission -authority to verify a billing or admin role and an injected verified tenant -context resolver. Client body fields cannot select a workspace. Gateway key +single-principal scope. The registered permission authority verifies an active +billing or admin workspace role using the same verified tenant context as +request admission. A top-up funds that principal in that workspace, rather than +the organisation as a whole. Client body fields cannot select a workspace. Gateway key scope enforcement also applies. Amounts are integer micro-USD, in whole cents: multiples of 10,000, from 500,000 @@ -91,8 +92,20 @@ Opening the URL allows payment at Stripe. Returning to the browser URL never credits a balance. The returned `pending` describes the checkout response, not a confirmed payment. +The ledger charges requests only when `CREDITS_ENABLED=true` and +`CREDITS_ENFORCEMENT` is `funded` (owners with a ledger revision above zero) or +`all` (every authenticated owner, including administrators). The default `off` +records top-ups without charging requests. Credit each principal before choosing +`all`; insufficient credit returns JSON 402 `credits_insufficient` before dispatch. + ## Processor evidence and reconciliation +Refunds and disputes append negative `adjust` entries with actor +`processor:stripe`, under the payment event's immutable operation ID. They may +make balance and available credit negative after credit has been spent; new +charged requests remain refused until the shortfall is covered. Only the verified +payment callback can create processor entries. + Configure Stripe to deliver these types to `POST /v1/payments/webhook`, and confirm them in the Stripe dashboard: @@ -100,8 +113,9 @@ confirm them in the Stripe dashboard: - `checkout.session.async_payment_succeeded`: credit the delayed payment. - `checkout.session.async_payment_failed`: record failure without credit. - `charge.refunded`: append the increase in cumulative refunded amount. -- `charge.dispute.created`: append a dispute compensation or hold through the - ledger callback, using the dispute ID as its stable operation identity. +- `charge.dispute.created`: append a negative processor adjustment through the + ledger callback, using the dispute ID as its stable operation identity. A later + dispute win requires an operator adjustment. The webhook requires no gateway key and is CSRF-exempt. It reads raw bytes before parsing JSON, verifies every supported `v1` signature candidate with @@ -123,10 +137,13 @@ original evidence hash or audit history. An atomic event claim precedes the immutable `PaymentEvent` callback. A completed replay returns 200 without invoking it again. Both session success types share one credit operation, cumulative refund notifications apply only -their delta, and repeat dispute IDs share one compensation operation. An event +their delta, and repeat dispute IDs share one adjustment operation. An event ID reused with different bytes returns 400 `payment_event_conflict`. -The callback is the only credit authority. It must apply its immutable +The registered credits callback applies verified top-ups and processor refunds +or disputes to the same per-principal workspace owner as `GET /v1/credits`. +Refunds after spending may make the balance negative and block new reservations. +The callback is the only payment credit authority. It applies its immutable idempotency ID atomically, including when recovering from a process crash after ledger application but before the payment record is finalized. A missing, denied, failing or unbound callback leaves `pending_credit` evidence and returns diff --git a/docs/saml-federation.md b/docs/saml-federation.md index f221e21e..204706a9 100644 --- a/docs/saml-federation.md +++ b/docs/saml-federation.md @@ -41,7 +41,7 @@ The nonce is claimed atomically once. A replay or expired durable request return After verification, an active `(issuer, subject)` link must name an existing, unrevoked, unexpired gateway account. No link returns HTTP 403 `saml_subject_unlinked`; the gateway never creates accounts from broker attributes. Assertion roles, administrator flags and organisation names grant nothing. The stored link supplies organisation, team and grants revision; the injected `IdentityAuthority` must return that exact `TenantContext`. Its `IdentityAssertion` uses digests for identifiers so arbitrary broker subjects fit the enterprise opaque-ID contract. Account names outside that contract use `account:` as principal ID, while session issuance still receives the actual account name. -The service requires a named dashboard session issuer that accepts `(account, TenantContext)` and returns `True` only after issuing the existing dashboard session. The default returns HTTP 503 `saml_session_unavailable`: API-key authentication has no separate session-creation function. The issuer must revalidate account permissions and populate the same session fields as dashboard login, using stored grants. The service uses the stored-link authority by default; an organisation authority can be injected to revalidate current membership. An unavailable authority returns HTTP 503 `saml_identity_unavailable`, and a scope mismatch returns HTTP 403 `saml_identity_denied`. +The service requires a named dashboard session issuer that accepts `(account, TenantContext)` and returns `True` only after issuing the existing dashboard session. The registered issuer calls the shared dashboard session function used by API-key login. The issuer must revalidate account permissions and populate the same session fields as dashboard login, using stored grants. The application composes the stored-link authority with the organisation authority to revalidate current active membership. An unavailable authority returns HTTP 503 `saml_identity_unavailable`, and a scope mismatch returns HTTP 403 `saml_identity_denied`. ## Administrator subject links @@ -57,7 +57,7 @@ PUT accepts `issuer`, `subject`, `account`, and optional `org_id`, `team_id`, `g ## Storage, audit and operational bounds -Apply `0035_saml_federation.sql` to D1 before enabling federation. It adds `saml_requests`, `saml_subject_links` and `saml_audit` and preserves existing accounts. The private handler is `handleSamlIdentityRequest` at `http://intelligence.internal/v1/managed-state/saml`. Its dispatcher and private client endpoint must be registered, and `register_saml_federation_routes(app, csrf)` must be mounted. The core login guard must include `SAML_PUBLIC_ENDPOINTS` in its public endpoint allowlist. The link routes retain their administrator guard. Until storage wiring is available, login fails closed. +Apply `0035_saml_federation.sql` to D1 before enabling federation. It adds `saml_requests`, `saml_subject_links` and `saml_audit` and preserves existing accounts. The private handler is `handleSamlIdentityRequest` at `http://intelligence.internal/v1/managed-state/saml`. The dispatcher, bounded private client and Flask routes are registered. `INTELLIGENCE_STORAGE_BACKEND=d1` is required. The core login guard includes `SAML_PUBLIC_ENDPOINTS`; link routes retain their administrator guard. Every private operation validates the required tables and columns before changing state. Missing storage or a missing migration returns JSON HTTP 503 `saml_storage_unavailable` without admitting login. There is no local-memory identity fallback or automatic replay of uncertain writes. Login audit records contain an issuer digest, linked account if known, outcome and time. `authorized` means signature, link and authority checks passed before session issuance; a later session failure adds its failure outcome. Link changes also record the operator's digest. Neither audit nor nonce rows contain tokens, raw subjects, nonce or request state. diff --git a/docs/scim-provisioning.md b/docs/scim-provisioning.md index f38af208..c7c6c138 100644 --- a/docs/scim-provisioning.md +++ b/docs/scim-provisioning.md @@ -8,7 +8,8 @@ the API and emits one warning without the value. Existing account creation is un ## Configuration and authentication Set `SCIM_ENABLED=true` only after applying the SCIM storage migration and configuring -the account and tenant authorities. `AUTH_STORAGE_BACKEND` and +the account and tenant authorities. The application registers the D1 store, +durable account service and bearer-scoped organisation team authority. `AUTH_STORAGE_BACKEND` and `INTELLIGENCE_STORAGE_BACKEND` must both be `d1`, and `CONFIG_REVISION_SYNC_ENABLED=true` must be enabled for the Worker and Container. Security revisions expire within `CONFIG_SECURITY_TTL_SECONDS` (default and maximum: diff --git a/docs/tenant-governance.md b/docs/tenant-governance.md index 336658ae..5595abe7 100644 --- a/docs/tenant-governance.md +++ b/docs/tenant-governance.md @@ -74,3 +74,13 @@ estimates, dispatch and finalization hooks. Flask uses the same domain through factories and database transactions. Neither adapter applies migrations automatically. Storage failure prevents new admissions; ambiguous settlement leaves the hold in place. Retention and any reconciliation of unknown holds require an operator policy. + +The application registers the verified tenant resolver and active membership-role +reader. Tool discovery and execution intersect durable key grants with every +configured organisation and team tool grant. + +Request accounting captures the verified workspace and its reservation authority +so streamed completion can settle all components after the request context ends. +Workspace budgets are checked before request credits. Shared cache, idempotency, +hosted-state, context-page and usage records include the non-empty verified +workspace namespace; legacy ownership bytes remain unchanged. diff --git a/docs/tenant-hierarchy.md b/docs/tenant-hierarchy.md index e6bff089..b7303549 100644 --- a/docs/tenant-hierarchy.md +++ b/docs/tenant-hierarchy.md @@ -99,11 +99,14 @@ replay an uncertain write. Standalone SQLite uses the configured account databas and serializes writes with an immediate transaction. Workspace identity does not introduce quotas, credits, payment collection, cost -allocation or provider discounts. Existing cache, idempotency, hosted-state and -usage ownership formats remain unchanged; a tenant binding alone does not isolate -those records. Consumers must incorporate the verified namespace before using -organisation isolation for shared records. Existing record ownership never moves +allocation or provider discounts. Cache, idempotency, hosted-state, context-page +and usage owners include the verified namespace when it is non-empty, isolating records +between workspaces. With organisations disabled, their ownership formats remain +byte-identical to the legacy formats. Existing record ownership never moves because an account selects a workspace. Audit and hierarchy rows are retained indefinitely; content-retention policies do not erase them. Deactivation keeps rows and continues to count toward the caps. There are no live-provider capability or metering guarantees from these APIs. + +The application registers the tenant authority and active membership-role reader +for enterprise identity, payment permissions and governance collaborators. diff --git a/package.json b/package.json index a2b61607..49cd2daa 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_wave7_worker.mjs tests/test_payment_billing_worker.mjs tests/test_credits_ledger_worker.mjs tests/test_scim_provisioning_worker.mjs tests/test_saml_federation_worker.mjs tests/test_tenant_governance_worker.mjs tests/test_tenant_hierarchy_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/route_helpers.py b/route_helpers.py index 1cf4fe83..e11ae17a 100644 --- a/route_helpers.py +++ b/route_helpers.py @@ -26,7 +26,7 @@ CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS" CORS_DEFAULT_HEADERS = ( - "Authorization, Content-Type, Accept, Origin, X-Requested-With, X-MultiLLM-Tool-Repair, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, " + "Authorization, Content-Type, Accept, Origin, X-Requested-With, X-MultiLLM-Tool-Repair, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, If-Match, " "X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, Anthropic-Version, " "Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, " "Idempotency-Key, X-MultiLLM-Priority, Prefer, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, OpenAI-Beta, OpenAI-Organization, " @@ -37,7 +37,7 @@ + ", ".join([*CLIENT_HEADER_NAMES.values(), *OPENCODE_CLIENT_HEADER_NAMES.values()]) ) CORS_EXPOSE_HEADERS = ( - "Retry-After, Location, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " + "Retry-After, Location, ETag, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, " "X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, " "X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, " "X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, " diff --git a/routes/chat_cache.py b/routes/chat_cache.py index d6168290..370476b7 100644 --- a/routes/chat_cache.py +++ b/routes/chat_cache.py @@ -122,6 +122,10 @@ def _principal() -> str | None: def cache_key(principal: str, path: str, payload: dict, *, policy: cache_policy.CachePolicy | None = None) -> str: canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) identity = (principal, path, canonical) + from services.tenant_hierarchy import tenant_namespace + namespace = tenant_namespace() + if namespace: + identity += (namespace,) if policy is not None: identity += (cache_policy.ISOLATED_REVISION, cache_policy.policy_digest(policy)) return hashlib.sha256("\x00".join(identity).encode("utf-8")).hexdigest() diff --git a/routes/core.py b/routes/core.py index 71dc5c4c..cd4d2013 100644 --- a/routes/core.py +++ b/routes/core.py @@ -448,7 +448,10 @@ def handle_redirects(): if request.headers.get("Authorization") or request_api_key(): return None - if request.endpoint in KNOWLEDGE_PUBLIC_ENDPOINTS | PRODUCT_PUBLIC_ENDPOINTS | STATUS_PUBLIC_ENDPOINTS or request.endpoint in [ + from routes.saml_federation import SAML_PUBLIC_ENDPOINTS + if (request.endpoint in KNOWLEDGE_PUBLIC_ENDPOINTS | PRODUCT_PUBLIC_ENDPOINTS | STATUS_PUBLIC_ENDPOINTS | SAML_PUBLIC_ENDPOINTS + or (request.endpoint or "").startswith("scim.") or request.endpoint in [ + "payment_webhook", "login", "login_access", "static_files", @@ -457,7 +460,7 @@ def handle_redirects(): "web_manifest", "service_worker", "apple_touch_icon", - ] or request.path.startswith("/static/"): + ] or request.path.startswith("/static/")): return None if not AuthService.is_authenticated(): diff --git a/routes/enterprise_preview.py b/routes/enterprise_preview.py index b2f9a6f8..64d2ec4c 100644 --- a/routes/enterprise_preview.py +++ b/routes/enterprise_preview.py @@ -8,7 +8,7 @@ def register_enterprise_preview_routes(app): - app.extensions["enterprise_adapters"] = contracts.register_enterprise_adapters() + app.extensions.setdefault("enterprise_adapters", contracts.register_enterprise_adapters()) def disabled_enterprise_preview(): if request.path != "/admin/enterprise/preview": diff --git a/routes/gateway_batches.py b/routes/gateway_batches.py index f0a50f8f..5d826fa2 100644 --- a/routes/gateway_batches.py +++ b/routes/gateway_batches.py @@ -9,6 +9,7 @@ from flask import Response, g, jsonify, request +from error_handlers import APIError from services import gateway_batches as batches from services.retention_policy import request_policy, resolve_policy from services.cost_service import CostService @@ -167,7 +168,7 @@ def pagination(): def execute_batch_item(app): - from services.media_signing import read_principal + from services.media_signing import read_principal, bind_principal_tenant from routes.media_batches import principal_user from services.auth_service import AuthService header = request.headers.get("Authorization", "") @@ -184,13 +185,17 @@ def execute_batch_item(app): if user is None or (started["batch"]["key_prefix"] and user.get("api_key_prefix") != started["batch"]["key_prefix"]): return {"status_code": 403, "body": {"error": {"code": "principal_rejected"}}, "cost_units": 0, "ambiguous": False} try: - return run_managed_item(app, user, started["item"], started["batch"]) + tenant = bind_principal_tenant(claims, user) + except APIError as error: + return {"status_code": error.status_code, "body": error.payload, "cost_units": 0, "ambiguous": False} + try: + return run_managed_item(app, user, started["item"], started["batch"], tenant=tenant) except Exception: # The managed accounting finalizer has already kept an uncertain hold. return {"status_code": 502, "body": {}, "cost_units": None, "ambiguous": True} -def run_managed_item(app, user, item, batch): +def run_managed_item(app, user, item, batch, *, tenant=None): from route_helpers import _authorize_api_scope, provider_from_request_path from services import request_accounting from services.gateway_extensions import after_authentication @@ -201,6 +206,11 @@ def run_managed_item(app, user, item, batch): with app.app_context(), app.test_request_context(item["url"], method="POST", json=item["body"], environ_base={"REMOTE_ADDR": batch["client_ip"]}, headers={"CF-Connecting-IP": batch["client_ip"], "Authorization": "BatchPrincipal managed-item"}): g.authenticated_user = user + from services.tenant_hierarchy import enabled as organisations_enabled + if organisations_enabled(): + if tenant is None: + raise batches.BatchError(503, "tenant_storage_unavailable", "The verified batch workspace is unavailable.") + g.verified_tenant = g.tenant_context = tenant g.gateway_batch_execution = True g.multillm_content_retention = resolve_policy(key_id=str(user.get("id") or user["username"]), key_hash=batch["key_hash"], route=item["url"]) diff --git a/routes/media_batches.py b/routes/media_batches.py index a63bf240..e69c46a2 100644 --- a/routes/media_batches.py +++ b/routes/media_batches.py @@ -295,6 +295,8 @@ def media_batch_items(): raise APIError("The batch owner's account no longer exists", status_code=403, payload={"error": "principal_rejected"}) g.authenticated_user = user + from services.media_signing import bind_principal_tenant + bind_principal_tenant(claims, user) # Key controls apply when an item runs, not only at submission: an allowlist or # budget changed since then is honoured, and each item is recorded in the ledger. started = time.perf_counter() diff --git a/routes/payments.py b/routes/payments.py index 45c061bc..957dddf3 100644 --- a/routes/payments.py +++ b/routes/payments.py @@ -27,6 +27,7 @@ def payment_gate(): return failed(error) if request.content_length is not None and request.content_length > payments.MAX_BODY: return failed(payments.PaymentError("invalid_payment_body", 400)) + request.max_content_length = payments.MAX_BODY return None app.before_request_funcs.setdefault(None, []).insert(0, payment_gate) @@ -45,9 +46,12 @@ def payment_checkout(): user = g.authenticated_user owner = str(user.get("id") or user.get("username") or "") context = context_resolver(user) if context_resolver else TenantContext(owner) + if context_resolver: + from services.credits_ledger import context_owner + owner = context_owner(context) if not request.is_json: raise payments.PaymentError("invalid_payment_checkout", 400) - raw = request.stream.read(payments.MAX_BODY + 1) + raw = request.get_data(cache=True) if len(raw) > payments.MAX_BODY: raise payments.PaymentError("invalid_payment_body", 400) try: @@ -65,7 +69,7 @@ def payment_checkout(): def payment_webhook(): try: # Signature validation always precedes parsing or reading identity. - raw = request.stream.read(payments.MAX_BODY + 1) + raw = request.get_data(cache=True) return jsonify(service.webhook(raw, request.headers.get("Stripe-Signature", ""))) except payments.PaymentError as error: return failed(error) diff --git a/routes/saml_federation.py b/routes/saml_federation.py index ca4f5cd4..e9c1f340 100644 --- a/routes/saml_federation.py +++ b/routes/saml_federation.py @@ -66,6 +66,8 @@ def saml_callback(): if set(values) != {"token", "state"} or any(len(values.getlist(key)) != 1 for key in values): raise SamlError("saml_request_invalid") browser_state_digest = session.get("saml_state_digest", "") + if not browser_state_digest and session.get("authenticated"): + raise SamlError("saml_assertion_replayed") original_session = deepcopy(dict(session)) try: service.complete(values["token"], values["state"], browser_state_digest) @@ -73,7 +75,6 @@ def saml_callback(): session.clear() session.update(original_session) raise - session["saml_state_digest"] = browser_state_digest return redirect("/", code=302) @app.route("/auth/saml/acs", methods=["GET", "POST"]) diff --git a/services/accounted_dispatch.py b/services/accounted_dispatch.py index 317ec971..dba61099 100644 --- a/services/accounted_dispatch.py +++ b/services/accounted_dispatch.py @@ -7,7 +7,7 @@ from error_handlers import APIError from services import key_controls, request_accounting, telemetry_export -from services.budget_service import BudgetService, budgeted +from services.budget_service import BudgetService from services.rate_limit_service import RateLimitService _ROUTING_FIELDS = ("multillm_model", "multillm_provider", "multillm_route_decision") @@ -17,7 +17,12 @@ def release_outer_accounting() -> None: """Subrequests replace the aggregate row and its reservation.""" context = getattr(g, "usage_context", None) if isinstance(context, request_accounting.UsageContext): - BudgetService.settle(context.reservation) + if not context.finished: + if context.credit_hold is not None or context.governance_store is not None: + request_accounting.release_before_dispatch() + else: + BudgetService.settle(context.reservation) + context.finished = True g.usage_context = None @@ -33,23 +38,24 @@ def accounted_dispatch(payload: dict, dispatch, *, kind: str, skip_rate: bool = context = request_accounting.UsageContext( kind=kind, models=[model], provider=None, user=user, started=time.perf_counter(), start_ns=time.time_ns(), input_tokens=input_tokens, output_tokens=output_tokens, units=units, - trace=telemetry_export.trace_context(request.headers.get("traceparent"))) + trace=telemetry_export.trace_context(request.headers.get("traceparent")), gateway_subrequest=True) request_id = getattr(g, "request_id", None) context.request_id = (request_id if isinstance(request_id, str) and request_accounting.REQUEST_ID.fullmatch(request_id) else None) context.path = "/v1/chat/completions" if kind == "chat" else "/v1/images/generations" - if budgeted(user): - decision = BudgetService.check_and_reserve( - user, request_accounting.estimate_cost([model], input_tokens, output_tokens, units)) - if not decision.allowed: - raise APIError(decision.message, decision.status_code, payload={"error": decision.error}) - context.reservation = decision.reservation + denied = request_accounting.reserve_attempt(context) + if denied is not None: + raise APIError(denied.json["message"], denied.status_code, payload={"error": denied.json["error"]}) routing = {name: getattr(g, name, None) for name in _ROUTING_FIELDS} for name in _ROUTING_FIELDS: setattr(g, name, None) response = None outer_subrequest = getattr(g, "gateway_subrequest", False) g.gateway_subrequest = True + outer_context = getattr(g, "usage_context", None) + owned = context.credit_hold is not None or context.governance_store is not None + if owned: + g.usage_context = context try: if not skip_rate: decision = RateLimitService.enforce_request( @@ -83,6 +89,8 @@ def accounted_dispatch(payload: dict, dispatch, *, kind: str, skip_rate: bool = request_accounting._record(context, getattr(error, "status_code", 502), None, None) raise finally: + if owned: + g.usage_context = outer_context g.gateway_subrequest = outer_subrequest for name, value in routing.items(): setattr(g, name, value) diff --git a/services/auth_service.py b/services/auth_service.py index 4b575062..81ca3b91 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -1091,6 +1091,17 @@ def authenticate_user(cls, username: str, api_key: str) -> bool: if not check_password_hash(user["api_key_hash"], api_key): return False + return cls.issue_dashboard_session(username, user=user) + + @classmethod + def issue_dashboard_session(cls, username: str, *, user: Optional[Dict[str, Any]] = None) -> bool: + """Issue a fresh session from the account's current stored permissions.""" + username = normalized_username(username) + if username is None: + return False + user = cls._load_user_by_username(username) if user is None else user + if not user or user.get("revoked_at") or key_controls.expired(user): + return False last_login = _utcnow() cls._update_login(username, last_login) user = cls._users[username] diff --git a/services/budget_service.py b/services/budget_service.py index 022aa65a..d1761105 100644 --- a/services/budget_service.py +++ b/services/budget_service.py @@ -54,6 +54,18 @@ def limits(user: Mapping[str, Any]) -> tuple[Optional[float], Optional[float]]: return user.get("daily_budget_usd"), user.get("monthly_budget_usd") +def principal_owner(user): + from flask import has_request_context + from services.tenant_hierarchy import current_tenant, principal_id, tenant_namespace + from services.credits_ledger import context_owner + principal = str(user.get("username") or user.get("id") or "") + if has_request_context(): + context = current_tenant() + if context.principal_id == principal_id(principal) and tenant_namespace(context): + return context_owner(context) + return principal + + def budgeted(user: Mapping[str, Any]) -> bool: from services.tenant_governance import workspace_context return workspace_context(user) is not None or any(limit is not None for limit in limits(user)) @@ -211,7 +223,7 @@ def check_and_reserve(cls, user: Mapping[str, Any], estimate_usd: Optional[float return BudgetDecision(True) if reservation_store.enabled(): return cls._reserve_durable(user, estimate_usd, now) - principal = str(user.get("username") or user.get("id") or "") + principal = principal_owner(user) period = periods(now) state = cls._state(principal) try: @@ -247,7 +259,7 @@ def check_and_reserve(cls, user: Mapping[str, Any], estimate_usd: Optional[float @classmethod def _durable_totals(cls, user, now=None): - principal = str(user.get("username") or user.get("id") or "") + principal = principal_owner(user) summary = reservation_store.get_store().summary(principal, now, reservation_store.review_seconds()) if not summary["day_seeded"] or not summary["month_seeded"]: state, period = cls._state(principal), periods(now) @@ -269,7 +281,7 @@ def _reserve_durable(cls, user, estimate, now=None): summary = cls._durable_totals(user, now) identity = uuid.uuid4().hex daily, monthly = limits(user) - reservation_store.get_store().reserve(identity, str(user.get("username") or user.get("id") or ""), + reservation_store.get_store().reserve(identity, principal_owner(user), estimate, daily, monthly, summary["spent_today_usd"], summary["spent_this_month_usd"], now) return BudgetDecision(True, reservation=identity) except reservation_store.ReservationError as error: @@ -281,10 +293,10 @@ def _reserve_durable(cls, user, estimate, now=None): message="Durable usage reservations are unavailable.") @classmethod - def mark_dispatched(cls, reservation): + def mark_dispatched(cls, reservation, *, governance_store=None): """Persist before provider submission; failure must prevent submission.""" from services.tenant_governance import dispatch - if dispatch(reservation): + if dispatch(reservation, authority=governance_store): return if not reservation or not reservation_store.enabled(): return @@ -296,10 +308,10 @@ def mark_dispatched(cls, reservation): raise reservation_store.ReservationError("reservation_conflict", 409) @classmethod - def complete(cls, reservation, row, *, cached=False): + def complete(cls, reservation, row, *, cached=False, governance_store=None): """Replace a hold only with measured cost; uncertain completion retains it.""" from services.tenant_governance import complete - if complete(reservation, row, cached=cached): + if complete(reservation, row, cached=cached, authority=governance_store): return if not reservation or not reservation_store.enabled(): return @@ -321,12 +333,12 @@ def complete(cls, reservation, row, *, cached=False): transition_id=uuid.uuid4().hex, **fields) @classmethod - def settle(cls, reservation: Optional[str], *, before_dispatch=False) -> None: + def settle(cls, reservation: Optional[str], *, before_dispatch=False, governance_store=None) -> None: """Release an in-flight estimate; the settled cost arrives through `record_cost`.""" if not reservation: return from services.tenant_governance import complete - if complete(reservation, {}, before_dispatch=before_dispatch): + if complete(reservation, {}, before_dispatch=before_dispatch, authority=governance_store): return if reservation_store.enabled(): if before_dispatch: @@ -364,7 +376,7 @@ def record_cost(cls, row: Mapping[str, Any]) -> None: def status(cls, user: Mapping[str, Any], now: Optional[datetime] = None) -> dict[str, Any]: """Spend and remaining budget for /v1/usage; reads durable totals when due.""" daily, monthly = limits(user) - principal = str(user.get("username") or user.get("id") or "") + principal = principal_owner(user) period = periods(now) state = cls._state(principal) available = True diff --git a/services/context_pages.py b/services/context_pages.py index 401c89f4..0f798755 100644 --- a/services/context_pages.py +++ b/services/context_pages.py @@ -10,7 +10,7 @@ import os import re import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from functools import lru_cache from typing import Any, Callable, Mapping @@ -290,6 +290,11 @@ def managed_authority(): authority = authorize() if not isinstance(authority, dict) or not {"scope", "retention_policy", "granted"} <= authority.keys(): raise ContextPageError("context_paging_authority_unavailable") + from services.shared_generation_cache import namespace_principal + scope = authority["scope"] + principal = namespace_principal(scope.principal) + if principal != scope.principal: + authority = {**authority, "scope": replace(scope, principal=hashlib.sha256(principal.encode()).hexdigest())} return authority user = getattr(g, "authenticated_user", None) or {} principal = authenticated_principal(user) @@ -301,6 +306,8 @@ def managed_authority(): revision = json.dumps({"grants": user.get("scopes"), "models": model_patterns(user), "secret": scan_mode(user), "retention": retention.revision}, sort_keys=True, separators=(",", ":")) + from services.shared_generation_cache import namespace_principal + principal = namespace_principal(principal) scope = PageScope(hashlib.sha256((principal + "\0" + key).encode()).hexdigest(), hashlib.sha256(session.encode()).hexdigest(), hashlib.sha256(revision.encode()).hexdigest()) return {"scope": scope, "retention_policy": retention, "granted": permits(user, "chat")} diff --git a/services/control_state_d1.py b/services/control_state_d1.py index 738f1056..f6bbbd32 100644 --- a/services/control_state_d1.py +++ b/services/control_state_d1.py @@ -37,7 +37,8 @@ def using_d1(): def call(endpoint, operation, **values): - return intelligence_d1_store.request_private_intelligence({"operation": operation, **values}, endpoint=endpoint) + version = {"version": 1} if endpoint in {"tenant_governance", "saml", "scim", "credits", "payments"} else {} + return intelligence_d1_store.request_private_intelligence({**version, "operation": operation, **values}, endpoint=endpoint) def cause(error): diff --git a/services/credits_admission.py b/services/credits_admission.py new file mode 100644 index 00000000..c2810102 --- /dev/null +++ b/services/credits_admission.py @@ -0,0 +1,213 @@ +"""Revision-bound request credits, separate from key and workspace budgets.""" +from __future__ import annotations + +import hashlib +import json +import os +import threading +import uuid +from contextvars import ContextVar +from dataclasses import dataclass, field +from decimal import Decimal, ROUND_CEILING +from typing import Any + +from flask import current_app, jsonify + +from services import credits_ledger +from services.enterprise_contract import AuthorityOperation, TenantContext, call_authority +from services.tenant_hierarchy import current_tenant + +_tariff_expectation: ContextVar[tuple | None] = ContextVar("request_credit_tariff", default=None) +REVISION_ATTEMPTS = 3 + + +class CreditAdmissionError(Exception): + def __init__(self, code="credits_unavailable", status=503): + super().__init__(code) + self.code, self.status = code, status + + +def error_response(error): + response = jsonify({"error": error.code, "message": "Request credit admission is unavailable." + if error.status == 503 else "This request exceeds available credits."}) + response.status_code = error.status + return response + + +def enforcement(): + return credits_ledger.enforcement() if credits_ledger.enabled() else "off" + + +def owner(context=None): + return credits_ledger.context_owner(context or current_tenant()) + + +def micro_usd(value, *, positive=False): + try: + number = Decimal(str(value)) * 1_000_000 + if isinstance(value, bool) or not number.is_finite() or number < 0: + raise ValueError() + amount = int(number.to_integral_value(rounding=ROUND_CEILING)) + credits_ledger.integer(amount, nonnegative=True) + if positive and amount == 0: + raise ValueError() + return amount + except (ValueError, ArithmeticError, credits_ledger.CreditsError): + raise CreditAdmissionError("credits_unpriced") from None + + +def tariff_revision(): + """The integer revision of the exact configured gateway price document.""" + raw = os.environ.get("MODEL_PRICING_USD_PER_MILLION", "") + return int(hashlib.sha256(raw.encode()).hexdigest()[:12], 16) + + +def _tariff(operation, phase): + expected = _tariff_expectation.get() + if expected is None or expected[:3] != (operation.operation_id, phase, operation.amount): + return None + return expected[3] + + +def register_credits_admission(app): + app.register_error_handler(CreditAdmissionError, error_response) + if not credits_ledger.enabled(): + return None + store = credits_ledger.open_store() + app.extensions["credits_admission_store"] = store + return credits_ledger.register_credit_authority(store, tariff=_tariff) + + +def _failure(error): + code = getattr(error, "code", None) + return CreditAdmissionError(code, 402 if code == "credits_insufficient" else 503) if code in { + "credits_insufficient", "credits_unpriced"} else CreditAdmissionError() + + +def _retry(store, context, scoped_id, operation_id, amount, apply): + """Only a refused revision permits retry; an uncertain write is never replayed.""" + for attempt in range(REVISION_ATTEMPTS): + try: + revision = store.read(owner(context), limit=1)["revision"] + operation = AuthorityOperation(context, scoped_id, revision, operation_id, amount) + return apply(operation) + except Exception as error: + if getattr(error, "code", None) == "credits_revision_mismatch" and attempt + 1 < REVISION_ATTEMPTS: + continue + raise _failure(error) from None + + +def operation_id(scoped_id, phase, transition_id=None): + identity = json.dumps([scoped_id, phase, transition_id], separators=(",", ":")) + return "credits:" + phase + ":" + hashlib.sha256(identity.encode()).hexdigest() + + +@dataclass +class CreditHold: + context: TenantContext + scoped_id: str + estimate: int + price_revision: int + store: Any + adapters: Any + dispatched: bool = False + terminal: bool = False + unknown: bool = False + lock: Any = field(default_factory=threading.Lock, repr=False) + + +def _authority(hold, phase, amount, *, transition_id=None, unknown=False): + identity = operation_id(hold.scoped_id, phase, transition_id) + token = _tariff_expectation.set((identity, phase, amount, None if unknown else hold.price_revision)) + try: + return _retry(hold.store, hold.context, hold.scoped_id, identity, amount, + lambda operation: call_authority(hold.adapters, "credit", phase, operation)) + finally: + _tariff_expectation.reset(token) + + +def reserve(estimate_usd, *, scoped_id=None, context=None): + mode = enforcement() + if mode == "off": + return None + context = context or current_tenant() + try: + store = current_app.extensions["credits_admission_store"] + adapters = current_app.extensions["enterprise_adapters"] + if mode == "funded" and store.read(owner(context), limit=1)["revision"] == 0: + return None + hold = CreditHold(context, scoped_id or uuid.uuid4().hex, micro_usd(estimate_usd, positive=True), + tariff_revision(), store, adapters) + _authority(hold, "reserve", hold.estimate) + return hold + except CreditAdmissionError: + raise + except Exception as error: + raise _failure(error) from None + + +def mark_dispatched(hold): + if hold is not None: + hold.dispatched = True + + +def release(hold, *, cancelled=False): + if hold is None: + return + with hold.lock: + if hold.terminal or (hold.dispatched and not cancelled): + return + identity = operation_id(hold.scoped_id, "release") + _retry(hold.store, hold.context, hold.scoped_id, identity, hold.estimate, + lambda operation: hold.store.append(owner(hold.context), "release", operation.amount, + operation.operation_id, operation.revision, scoped_id=operation.scoped_id)) + hold.terminal = True + + +def complete(hold, row, *, cached=False, before_dispatch=False, transition_id=None): + if hold is None: + return + if before_dispatch or cached or (not hold.dispatched and row.get("status", 200) >= 400): + release(hold) + return + with hold.lock: + if hold.terminal: + return + measured = row.get("cost_usd") is not None and row.get("cost_basis") == "usage" + if not measured and hold.unknown: + return + _authority(hold, "reconcile" if transition_id or not measured else "commit", + micro_usd(row["cost_usd"]) if measured else 0, + transition_id=transition_id, unknown=not measured) + hold.terminal, hold.unknown = measured, not measured + + +def reconcile_reservation(row, transition_id): + """Recover the credits attempt from immutable ledger evidence after a restart.""" + if not credits_ledger.enabled(): + return + try: + principal = row["principal"] + if principal.startswith("tenant:"): + org, team, principal = json.loads(principal[7:]) + context = TenantContext(principal, org, team) + else: + context = TenantContext(principal) + store = current_app.extensions["credits_admission_store"] + cursor, entries = "0", [] + while True: + page = store.read(owner(context), cursor=cursor) + entries.extend(entry for entry in page["entries"] if entry["scoped_id"] == row["id"]) + cursor = page["next_cursor"] + if cursor is None: + break + reserve_entry = next((entry for entry in entries if entry["kind"] == "reserve" and entry["held_delta"] > 0), None) + if reserve_entry is None or any(entry["kind"] in {"commit", "release"} for entry in entries): + return + hold = CreditHold(context, row["id"], reserve_entry["amount_microusd"], reserve_entry["tariff_revision"], + store, current_app.extensions["enterprise_adapters"], dispatched=True) + complete(hold, {"cost_usd": row["cost_usd"], "cost_basis": "usage"}, transition_id=transition_id) + except CreditAdmissionError: + raise + except Exception as error: + raise _failure(error) from None diff --git a/services/credits_ledger.py b/services/credits_ledger.py index 63a85e71..20b62464 100644 --- a/services/credits_ledger.py +++ b/services/credits_ledger.py @@ -67,6 +67,16 @@ def integer(value, *, nonnegative=False): return value +def enforcement(env: Mapping | None = None) -> str: + values = os.environ if env is None else env + raw = values.get('CREDITS_ENFORCEMENT', '') + mode = raw.strip().lower() if isinstance(raw, str) else 'invalid' + if mode not in {'', 'off', 'funded', 'all'}: + _warn('CREDITS_ENFORCEMENT') + mode = 'off' + return (mode or 'off') if enabled(values) else 'off' + + def label(value): if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_:.\-]{1,128}', value): raise CreditsError('invalid_credits_request', 400) @@ -105,6 +115,8 @@ def request_fields(owner, kind, amount_microusd, operation_id, revision, *, scop for value in (scoped_id, reference, actor): if value is not None: label(value) + if actor is not None and actor.startswith('processor:'): + raise CreditsError('invalid_credits_request', 400) if kind in {'reserve', 'commit', 'release'}: label(scoped_id) integer(amount_microusd, nonnegative=True) @@ -132,6 +144,27 @@ def request_fields(owner, kind, amount_microusd, operation_id, revision, *, scop actor=actor, reason=reason, unknown=unknown) +def payment_fields(owner, event_kind, amount_microusd, operation_id, revision, *, scoped_id, processor_id): + if event_kind not in {'credit', 'refund', 'dispute'} or processor_id != 'stripe': + raise CreditsError('invalid_credits_request', 400) + integer(amount_microusd, nonnegative=True) + label(scoped_id) + fields = request_fields(owner, 'credit' if event_kind == 'credit' else 'adjust', + amount_microusd if event_kind == 'credit' else -amount_microusd, + operation_id, revision, scoped_id=scoped_id, actor=processor_id, + reason=event_kind) + fields['actor'] = 'processor:' + processor_id + return fields + + +def same_payment(existing, document): + """Payment evidence is immutable even when a retry observes a later ledger revision.""" + before, after = json.loads(existing), json.loads(document) + before.pop('revision', None) + after.pop('revision', None) + return before == after + + def deltas(fields, entries): kind, amount = fields['kind'], fields['amount_microusd'] balance_delta, held_delta = 0, 0 @@ -202,11 +235,19 @@ def _run(self, action, *, check=True): def append(self, owner, kind, amount_microusd, operation_id, revision, **kwargs): fields = request_fields(owner, kind, amount_microusd, operation_id, revision, **kwargs) + return self._append(fields) + + def append_payment(self, owner, event_kind, amount_microusd, operation_id, revision, **kwargs): + return self._append(payment_fields(owner, event_kind, amount_microusd, operation_id, revision, **kwargs), payment=True) + + def _append(self, fields, *, payment=False): + owner, kind, amount_microusd, operation_id, revision = (fields[name] for name in + ('owner', 'kind', 'amount_microusd', 'operation_id', 'revision')) document = json.dumps(fields, sort_keys=True, ensure_ascii=False, separators=(',', ':')) def write(db): existing = db.execute('SELECT * FROM credits_entries WHERE owner=? AND operation_id=?', (owner, operation_id)).fetchone() if existing: - if existing['document'] != document: + if existing['document'] != document and not (payment and same_payment(existing['document'], document)): raise CreditsError('credits_conflict', 409) return public_entry(existing) row = db.execute('SELECT * FROM credits_balances WHERE owner=?', (owner,)).fetchone() @@ -221,7 +262,7 @@ def write(db): integer(revision + 1, nonnegative=True) available = balance + bd - held - hd reservation = kind == 'reserve' and not fields['unknown'] - enforced_spend = reservation or (kind == 'adjust' and amount_microusd < 0) + enforced_spend = reservation or (kind == 'adjust' and amount_microusd < 0 and not payment) if enforced_spend and (available < 0 or (reservation and balance - held <= 0)): raise CreditsError('credits_insufficient', 409) db.execute('INSERT OR IGNORE INTO credits_balances(owner) VALUES (?)', (owner,)) @@ -284,6 +325,11 @@ def append(self, owner, kind, amount_microusd, operation_id, revision, **kwargs) fields = request_fields(owner, kind, amount_microusd, operation_id, revision, **kwargs) return self._call('append', **fields) + def append_payment(self, owner, event_kind, amount_microusd, operation_id, revision, **kwargs): + payment_fields(owner, event_kind, amount_microusd, operation_id, revision, **kwargs) + return self._call('payment_append', owner=owner, event_kind=event_kind, amount_microusd=amount_microusd, + operation_id=operation_id, revision=revision, **kwargs) + def read(self, owner, *, cursor='0', limit=100): owner_id(owner) pagination(cursor, limit) @@ -318,14 +364,18 @@ def _validate_entry(value): def _validate_private_result(operation, result, values): - name = 'entry' if operation == 'append' else 'summary' + append = operation in {'append', 'payment_append'} + name = 'entry' if append else 'summary' if set(result) != {'version', name}: raise CreditsError() value = result[name] - if operation == 'append': + if append: _validate_entry(value) - if (any(value[key] != values[key] for key in ('operation_id', 'kind', 'amount_microusd', 'scoped_id', 'tariff_revision')) - or value['revision'] != values['revision'] + 1): + expected = (payment_fields(**values) if operation == 'payment_append' else values) + valid_revision = (0 < value['revision'] <= values['revision'] + 1 if operation == 'payment_append' + else value['revision'] == values['revision'] + 1) + if (any(value[key] != expected[key] for key in ('operation_id', 'kind', 'amount_microusd', 'scoped_id', 'tariff_revision')) + or not valid_revision): raise CreditsError() return value if not isinstance(value, dict) or set(value) != {'currency', 'balance_microusd', 'held_microusd', 'available_microusd', 'revision', 'entries', 'next_cursor'}: diff --git a/services/credits_payments.py b/services/credits_payments.py new file mode 100644 index 00000000..31c58daa --- /dev/null +++ b/services/credits_payments.py @@ -0,0 +1,33 @@ +"""Verified processor evidence credits the same owner used by request admission.""" +from services import credits_ledger as ledger +from services.enterprise_contract import AuthorityResult, PaymentEvent + + +def payment_callback(store=None): + """Construct a lazy callback; registration does not open or query storage.""" + def apply(event): + if type(event) is not PaymentEvent or event.verified is not True: + raise TypeError('Verified PaymentEvent required') + if not ledger.enabled(): + raise ledger.CreditsError('credits_disabled', 404) + if event.currency != 'USD': + raise ledger.CreditsError('invalid_credits_request', 400) + from services.credits_admission import CreditAdmissionError, _retry + authority = store if store is not None else ledger.open_store() + owner = ledger.context_owner(event.context) + + def append(operation): + try: + return authority.append_payment(owner, event.kind, operation.amount, + operation.operation_id, operation.revision, scoped_id=operation.scoped_id, + processor_id=event.processor_id) + except ledger.CreditsError as error: + if error.code == 'credits_revision_mismatch' and error.status != 412: + raise CreditAdmissionError() from None + raise + + _retry(authority, event.context, event.scoped_id, event.idempotency_id, event.amount, + append) + return AuthorityResult(event.context, event.scoped_id, event.revision + 1, + event.idempotency_id, True) + return apply diff --git a/services/gateway_extensions.py b/services/gateway_extensions.py index 7652d26f..6e015206 100644 --- a/services/gateway_extensions.py +++ b/services/gateway_extensions.py @@ -192,6 +192,8 @@ def require_config_security_freshness(): protected = (is_api_request_path(request.path) and request.path.rstrip("/") not in {"/health", "/healthz"} or request.endpoint == "proxy" or request.path in {"/mcp", "/mcp/", "/googleai/chat/completions", "/api/backends/chat-completions/generate"}) + if request.endpoint == "payment_webhook": + protected = False if request.method != "OPTIONS" and protected and not sync.security_ready(): return jsonify({"error": {"code": "config_security_stale", "message": "Security configuration freshness could not be verified"}}), 503, {"Cache-Control": "no-store"} @@ -218,3 +220,88 @@ def config_revision_status(): register_alert_routes(app) app.extensions["gateway_alert_observer"] = observe app.extensions["gateway_extensions_registered"] = True + + +def register_enterprise_features(app, csrf): + """Compose fixed enterprise authorities once without probing disabled storage.""" + if app.extensions.get("enterprise_features_registered"): + return + from functools import partial + from services import credits_ledger, tenant_governance, tenant_hierarchy + from services.auth_service import AuthService + from services.credits_admission import register_credits_admission + from services.credits_payments import payment_callback + from services.enterprise_contract import legacy_tenant, register_enterprise_adapters + from services.saml_federation import SamlFederation, load_config, stored_link_authority + tenant = tenant_hierarchy.tenant_authority if tenant_hierarchy.enabled() else legacy_tenant + + def identity(assertion, operation): + stored_link_authority(assertion, operation) + return tenant(operation) + + saml = SamlFederation(load_config(callback_url=app.config.get("SAML_CALLBACK_URL")), identity_authority=identity) + callback = payment_callback() + app.extensions["enterprise_adapters"] = register_enterprise_adapters( + tenant=tenant, identity=identity if saml.config.enabled else None, + credit=register_credits_admission(app), payment=callback if credits_ledger.enabled() else None) + app.extensions["saml_federation"] = saml + tenant_governance.register_governance_collaborators( + tenant_resolver=lambda user: tenant_hierarchy.current_tenant(), + membership_role=tenant_hierarchy.membership_role, app=app) + from routes.saml_federation import register_saml_federation_routes + from routes.scim import register_scim_routes + from routes.credits import register_credits_routes + from routes.payments import register_payment_routes + from services.intelligence_d1_store import request_private_intelligence + from services.scim_provisioning import D1ScimStore, ScimService + scim_store = D1ScimStore(partial(request_private_intelligence, endpoint="scim")) + register_saml_federation_routes(app, csrf) + register_scim_routes(app, service=ScimService(scim_store, AuthService, + tenant_authority=partial(tenant_hierarchy.scim_team_authority, scim_store=scim_store))) + register_credits_routes(app, csrf) + register_payment_routes(app, csrf, service=_payment_service(callback), + context_resolver=lambda user: tenant_hierarchy.current_tenant()) + if tenant_governance.enabled(): + _register_governance_tools(app) + app.extensions["enterprise_features_registered"] = True + + +def _payment_service(callback): + import os + from functools import partial + from services.intelligence_d1_store import request_private_intelligence + from services.payment_billing import D1PaymentStore, PaymentBilling, SqlPaymentStore + from services.sqlite_store import storage_path + backend = os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip().lower() + store = D1PaymentStore(partial(request_private_intelligence, endpoint="payments")) if backend == "d1" else ( + SqlPaymentStore(storage_path("USAGE_DB_PATH", "usage.sqlite3")) if backend in {"", "sqlite"} else None) + return PaymentBilling(store=store, callback=callback, permission=_payment_permission) + + +def _payment_permission(context, owner): + from services.credits_ledger import context_owner + from services.tenant_hierarchy import enabled, membership_role + if owner != context_owner(context): + return False + if context.org_id is None: + return owner == context.principal_id + return enabled() and membership_role(context.principal_id, context.org_id) in {"billing", "admin"} + + +def _register_governance_tools(app): + from flask import g + from services.deferred_tools import D1GrantReader, DeferredTools + from services.tenant_governance import grant_allowed + reader = D1GrantReader() + + def grants(principal): + rows = reader(principal) + user = getattr(g, "authenticated_user", None) + if not isinstance(rows, list): + return rows + return [{**row, "allowed": int(grant_allowed(bool(row.get("allowed")), row.get("tool_name"), + user=user, kind="tools"))} if isinstance(row, dict) + and type(row.get("allowed")) is int and row["allowed"] in {0, 1} else row + for row in rows] + + app.extensions["deferred_tools"] = DeferredTools(grants) diff --git a/services/hedged_requests.py b/services/hedged_requests.py index 9e9c9403..7f41092e 100644 --- a/services/hedged_requests.py +++ b/services/hedged_requests.py @@ -138,7 +138,8 @@ def _prepare_holds(models, payload): raise APIError("Configure both candidate prices before hedging this route.", 503, {"error": "unpriced_reservation"}) try: - held = reservation_store.get_store().get(outer.reservation) + held = ({"state": "reserved" if not outer.dispatched else "dispatched", "estimate_usd": outer.estimate} + if outer.governance_store is not None else reservation_store.get_store().get(outer.reservation)) except reservation_store.ReservationError as error: raise APIError("The durable usage reservation is unavailable.", 503, {"error": error.code}) from error @@ -151,7 +152,13 @@ def _prepare_holds(models, payload): raise APIError(decision.message, decision.status_code, {"error": decision.error}) if decision.reservation is None: return None - return [outer.reservation, decision.reservation] + from services import credits_admission + try: + secondary_credit = credits_admission.reserve(costs[1], scoped_id=decision.reservation, context=outer.tenant) + except credits_admission.CreditAdmissionError: + BudgetService.settle(decision.reservation, before_dispatch=True, governance_store=outer.governance_store) + raise + return [(outer.reservation, outer.credit_hold), (decision.reservation, secondary_credit)] def _useful_response(response): @@ -178,7 +185,9 @@ def _useful_response(response): class _Attempt: def __init__(self, model, position, hold, outer, expires, clock, changed): self.model, self.position = model, position - self.context = replace(outer, models=[model], selected=model, reservation=hold, + reservation, credit_hold = hold + self.context = replace(outer, models=[model], selected=model, reservation=reservation, credit_hold=credit_hold, + dispatched=False, finished=False, ambiguous=False, output_tokens=outer.output_tokens) self.owner = RequestCancellation() self.deadline = Deadline(expires, clock) @@ -217,7 +226,10 @@ def before_submission(self): def _dispatch_started(self): try: - BudgetService.mark_dispatched(self.context.reservation) + BudgetService.mark_dispatched(self.context.reservation, + **({"governance_store": self.context.governance_store} if self.context.governance_store is not None else {})) + accounting.credits_admission.mark_dispatched(self.context.credit_hold) + self.context.dispatched = True except reservation_store.ReservationError as error: raise APIError("The durable dispatch boundary is unavailable.", 503, {"error": error.code}) from error @@ -263,9 +275,14 @@ def _settle(self): self.context.ambiguous = self.context.ambiguous or ( self.usage is None and (self.submissions > 0 or self.bridge.handed_off)) if not self.bridge.handed_off and not self.submissions: - BudgetService.settle(self.context.reservation, before_dispatch=True) + accounting.credits_admission.release(self.context.credit_hold) + BudgetService.settle(self.context.reservation, before_dispatch=True, governance_store=self.context.governance_store) self.context.finished = True return + if not self.accepted and self.stopped.is_set(): + accounting.credits_admission.release(self.context.credit_hold, cancelled=True) + accounting.tenant_governance.complete(self.context.reservation, {}, cancelled=True, + authority=self.context.governance_store) status = self.response.status_code if self.response is not None else getattr(self.error, "status_code", 502) accounting._record(self.context, status, self.usage, None) @@ -292,7 +309,7 @@ def run(self, dispatch, body, decision, snapshot, turn, affinity, finished): g.managed_idempotency_claim = None g.gateway_admission_lease = None g.hedged_attempt = self - with isolated_managed_attempt(turn) as local_turn, activate_scope(affinity): + with isolated_managed_attempt(turn, self.context.reservation) as local_turn, activate_scope(affinity): try: self.deadline.check() if self.stopped.is_set(): @@ -449,7 +466,9 @@ def execute(self, policy, dispatch, decision): if attempt.thread is not None and attempt.started: attempt.thread.join() elif not attempt.started: - BudgetService.settle(attempt.context.reservation, before_dispatch=True) + accounting.credits_admission.release(attempt.context.credit_hold) + BudgetService.settle(attempt.context.reservation, before_dispatch=True, + governance_store=attempt.context.governance_store) if attempt.lease is not None: attempt.lease.release() if selected.error is not None: diff --git a/services/idempotency_store.py b/services/idempotency_store.py index dc828a27..ffb24124 100644 --- a/services/idempotency_store.py +++ b/services/idempotency_store.py @@ -60,6 +60,8 @@ def request_fingerprint(principal, method, path, key, body, revision): raise GatewayError("invalid_idempotency_key", "Idempotency-Key requires 1 to 128 safe ASCII characters.") if not isinstance(principal, str) or not principal or len(principal) > 256: raise unavailable() + from services.shared_generation_cache import namespace_principal + principal = namespace_principal(principal) try: scope = hashlib.sha256(_canonical([principal, method, path, key])).hexdigest() digest = hashlib.sha256(_canonical([body, revision])).hexdigest() diff --git a/services/intelligence_d1_store.py b/services/intelligence_d1_store.py index bb6a9e87..0c737918 100644 --- a/services/intelligence_d1_store.py +++ b/services/intelligence_d1_store.py @@ -38,13 +38,22 @@ "context-pages": "http://intelligence.internal/v1/managed-state/context-pages", "learned-cooldown": "http://intelligence.internal/v1/state/learned-cooldown", "usage-receipts": "http://intelligence.internal/v1/managed-state/usage-receipts", + "tenant_governance": "http://intelligence.internal/v1/tenant-governance", + "saml": "http://intelligence.internal/v1/managed-state/saml", + "scim": "http://intelligence.internal/v1/managed-state/scim", + "credits": "http://intelligence.internal/v1/managed-state/credits", + "payments": "http://intelligence.internal/v1/managed-state/payments", } _MAX_BYTES = 262144 _ENDPOINT_MAX_BYTES = {"context-pages": 2 * 1024 * 1024, "learned-cooldown": 4096, "usage-receipts": 128 * 1024} +_ENDPOINT_MAX_BYTES.update({"tenant_governance": 32768, "saml": 8192, "scim": 131072, + "credits": 16384, "payments": 65536}) _RESPONSE_MAX_BYTES = {_ENDPOINTS["context-pages"]: _ENDPOINT_MAX_BYTES["context-pages"], _ENDPOINTS["learned-cooldown"]: _ENDPOINT_MAX_BYTES["learned-cooldown"], _ENDPOINTS["usage-receipts"]: _ENDPOINT_MAX_BYTES["usage-receipts"], _ENDPOINTS["cascades"]: 524288} +_RESPONSE_MAX_BYTES.update({_ENDPOINTS[name]: _ENDPOINT_MAX_BYTES[name] + for name in ("tenant_governance", "saml", "scim", "credits", "payments")}) _TIMEOUT = (2, 3) _DEADLINE_SECONDS = 5 # Every authenticated request waits on an account lookup, and the first call after diff --git a/services/managed_dispatch.py b/services/managed_dispatch.py index 2d53b0b1..5e162390 100644 --- a/services/managed_dispatch.py +++ b/services/managed_dispatch.py @@ -4,6 +4,8 @@ import json import time from contextlib import contextmanager +from contextvars import ContextVar +import uuid from dataclasses import replace from typing import Callable @@ -19,15 +21,24 @@ Outcome = Callable[[], UpstreamOutcome] +_attempt_identity = ContextVar("managed_attempt_identity", default=None) + + +def current_attempt_id(): + return _attempt_identity.get() + + @contextmanager -def isolated_managed_attempt(turn): +def isolated_managed_attempt(turn, scoped_id=None): """Keep parallel attempt finalizers out of the parent managed turn.""" from services.managed_turn import _turn local = replace(turn, finalizers=[], canary=None, canary_finalized=False, pages=list(turn.pages), paged_candidates=dict(turn.paged_candidates)) if turn else None token = _turn.set(local) + attempt_token = _attempt_identity.set(scoped_id or uuid.uuid4().hex) try: yield local finally: + _attempt_identity.reset(attempt_token) _turn.reset(token) diff --git a/services/managed_turn.py b/services/managed_turn.py index 709ebca0..cd0a3e98 100644 --- a/services/managed_turn.py +++ b/services/managed_turn.py @@ -346,8 +346,12 @@ def submit(): raise GatewayError("cancelled", "The caller cancelled the request.", 499) if context is not None: from services.budget_service import BudgetService + from services import credits_admission - BudgetService.mark_dispatched(context.reservation) + BudgetService.mark_dispatched(context.reservation, + **({"governance_store": context.governance_store} if context.governance_store is not None else {})) + credits_admission.mark_dispatched(context.credit_hold) + context.dispatched = True return deadline, submit diff --git a/services/media_signing.py b/services/media_signing.py index 050555dd..1c8c54c2 100644 --- a/services/media_signing.py +++ b/services/media_signing.py @@ -72,6 +72,15 @@ def verify_file_link(file_id: str, expires: object, signature: object, now: floa def issue_principal(kind: str, subject: str, owner: str, ttl: int) -> str: """A capability for one job, carrying the owner's identity instead of their key.""" claims = {"k": kind, "s": subject, "o": owner, "e": int(time.time()) + ttl} + from flask import g, has_request_context + from services.tenant_hierarchy import enabled, principal_id + if enabled(): + context = getattr(g, "tenant_context", None) if has_request_context() else None + if context is None or context.principal_id != principal_id(owner): + raise APIError("The verified job workspace is unavailable", 503, + payload={"error": "tenant_storage_unavailable"}) + claims["tenant"] = {name: getattr(context, name) for name in + ("principal_id", "org_id", "team_id", "grants_revision")} payload = _b64url(json.dumps(claims, separators=(",", ":"), sort_keys=True).encode("utf-8")) return f"{payload}.{_b64url(_mac('principal', payload))}" @@ -93,6 +102,24 @@ def read_principal(token: object, kind: str) -> dict: return claims +def bind_principal_tenant(claims: dict, user: dict): + """Bind a signed submission workspace without resolving today's binding.""" + from flask import g + from services.enterprise_contract import TenantContext + from services.tenant_hierarchy import enabled, principal_id + if not enabled(): + return None + try: + context = TenantContext(**claims["tenant"]) + if context.principal_id != principal_id(user.get("username") or user.get("id") or ""): + raise ValueError() + except (KeyError, TypeError, ValueError): + raise APIError("The verified job workspace is unavailable", 503, + payload={"error": "tenant_storage_unavailable"}) from None + g.verified_tenant = g.tenant_context = context + return context + + def webhook_secret_bytes(owner: str) -> bytes: return _mac("webhook", owner) diff --git a/services/payment_billing.py b/services/payment_billing.py index 26704304..77d8ea0f 100644 --- a/services/payment_billing.py +++ b/services/payment_billing.py @@ -288,7 +288,10 @@ def call(self, operation, **data): return reply["result"] except PaymentError: raise - except Exception: + except Exception as error: + from services.intelligence_d1_store import PrivateIntelligenceError + if isinstance(error, PrivateIntelligenceError): + raise PaymentError(error.code, error.status) from None raise PaymentError() from None diff --git a/services/request_accounting.py b/services/request_accounting.py index 73d0006e..e026251e 100644 --- a/services/request_accounting.py +++ b/services/request_accounting.py @@ -14,6 +14,7 @@ import logging import re import time +import uuid from dataclasses import dataclass, field, replace from datetime import datetime, timezone from typing import Any, Iterable, Optional @@ -23,6 +24,7 @@ from error_handlers import APIError from services import key_controls, prompt_cache_cost, reservation_store, telemetry_export, usage_ledger, stream_cost_breaker +from services import credits_admission, tenant_governance from services.budget_service import BudgetService, budgeted from services.cost_service import CostService from services.usage_types import StreamUsageObserver, UsageObservation @@ -94,6 +96,11 @@ class UsageContext: cached: bool = False ambiguous: bool = False stream_cost: Any = None + tenant: Any = None + credit_hold: Any = None + governance_store: Any = None + dispatched: bool = False + gateway_subrequest: bool = False def classify() -> Optional[str]: @@ -226,7 +233,10 @@ def mark_dispatched(): """Injected pre-submission hook for managed request owners.""" context = getattr(g, "usage_context", None) if isinstance(context, UsageContext): - BudgetService.mark_dispatched(context.reservation) + BudgetService.mark_dispatched(context.reservation, + **({"governance_store": context.governance_store} if context.governance_store is not None else {})) + credits_admission.mark_dispatched(context.credit_hold) + context.dispatched = True def estimate_cost(models: list[str], input_tokens: int, output_tokens: int, units: int) -> float: @@ -253,6 +263,8 @@ def check_key_controls(user: dict) -> Optional[Response]: def begin() -> Optional[Response]: """Enforce the model allowlist and budget before dispatch; None admits the request.""" + if isinstance(getattr(g, "usage_context", None), UsageContext): + return None kind = classify() user = getattr(g, "authenticated_user", None) if kind is None or not isinstance(user, dict): @@ -270,6 +282,8 @@ def begin() -> Optional[Response]: started=getattr(g, "request_started_at", None) or time.perf_counter(), start_ns=time.time_ns(), units=_image_units(kind, payload), trace=telemetry_export.trace_context(request.headers.get("traceparent"))) + from services.tenant_hierarchy import current_tenant + context.tenant = current_tenant() context.input_tokens, context.output_tokens = _token_estimates(payload) if payload and payload.get("stream") is True and stream_cost_breaker.enabled() and ( user.get(stream_cost_breaker.CAP_FIELD) is not None or user.get("max_stream_cost_usd") is not None): @@ -281,17 +295,44 @@ def begin() -> Optional[Response]: user, qualified, max(context.input_tokens, stream_cost_breaker.input_bound(payload)), protocol) except APIError as error: return _error(error.status_code, error.payload["error"], "Stream cost enforcement refused this request.") - if budgeted(user): - qualified = [_qualified(model, provider) for model in models] + denied = reserve_attempt(context) + if denied is not None: + return denied + g.usage_context = context + return None + + +def reserve_attempt(context): + """Reserve each gateway-owned provider attempt before any submission.""" + from services.managed_dispatch import current_attempt_id + from services.tenant_hierarchy import current_tenant, enabled as organisations_enabled + if organisations_enabled() and not hasattr(g, "tenant_context"): + return _error(503, "tenant_storage_unavailable", "The verified request workspace is unavailable.") + user = context.user + context.tenant = context.tenant or current_tenant() + governance = tenant_governance.workspace_context(user) + charged = credits_admission.enforcement() != "off" + qualified = [_qualified(model, context.provider) for model in context.models] + estimate = None + if budgeted(user) or charged: estimate = (reservation_price(qualified, context.input_tokens, context.output_tokens, context.units) - if reservation_store.enabled() else estimate_cost(qualified, context.input_tokens, context.output_tokens, context.units)) + if (reservation_store.enabled() and not context.gateway_subrequest) or governance is not None or charged else + estimate_cost(qualified, context.input_tokens, context.output_tokens, context.units)) + if budgeted(user): context.estimate = estimate or 0.0 decision = BudgetService.check_and_reserve(user, estimate) if not decision.allowed: return _error(decision.status_code, decision.error, decision.message, decision.retry_after, **({"budget": decision.details} if decision.details else {})) context.reservation = decision.reservation - g.usage_context = context + if tenant_governance.governance_reservation(context.reservation): + context.governance_store = tenant_governance.store() + if charged: + try: + context.credit_hold = credits_admission.reserve(estimate, scoped_id=context.reservation or (uuid.uuid4().hex if context.gateway_subrequest else current_attempt_id()), context=context.tenant) + except credits_admission.CreditAdmissionError as error: + BudgetService.settle(context.reservation, before_dispatch=True, **({"governance_store": context.governance_store} if context.governance_store is not None else {})) + return credits_admission.error_response(error) return None @@ -391,8 +432,8 @@ def cancellation_observer(): if not isinstance(context, UsageContext): return None - if reservation_store.enabled(): - BudgetService.mark_dispatched(context.reservation) + if reservation_store.enabled() or context.governance_store is not None or context.credit_hold is not None: + mark_dispatched() def observe(outcome): if outcome.ambiguous and not context.finished: @@ -402,8 +443,24 @@ def observe(outcome): return observe +def release_before_dispatch(): + """Release admission after a later hook refuses the request, exactly once.""" + context = getattr(g, "usage_context", None) + if not isinstance(context, UsageContext) or context.finished or context.dispatched: + return + credits_admission.release(context.credit_hold) + BudgetService.settle(context.reservation, before_dispatch=True, **({"governance_store": context.governance_store} if context.governance_store is not None else {})) + context.finished = True + + def _record_budget(context, row, budget_row=None): - if reservation_store.enabled() and context.reservation: + credits_admission.complete(context.credit_hold, row, cached=context.cached) + if tenant_governance.governance_reservation(context.reservation): + if not context.dispatched and row.get("status", 200) >= 400: + BudgetService.settle(context.reservation, before_dispatch=True, governance_store=context.governance_store) + BudgetService.complete(context.reservation, row, cached=context.cached, + **({"governance_store": context.governance_store} if context.governance_store is not None else {})) + elif reservation_store.enabled() and context.reservation: held = reservation_store.get_store().get(context.reservation) # Admission owns the UTC budget period, including completion after midnight. row = {**row, "at": datetime.fromtimestamp(held["created_at"] / 1000, timezone.utc).isoformat( @@ -423,10 +480,8 @@ def _record(context: UsageContext, status: int, usage: Optional[UsageObservation if context.ambiguous: status, usage, units = 499, None, None if context.path in DEFERRED_PATHS: - if reservation_store.enabled(): - BudgetService.settle(context.reservation, before_dispatch=True) - else: - BudgetService.settle(context.reservation) + credits_admission.release(context.credit_hold) + BudgetService.settle(context.reservation, before_dispatch=True, **({"governance_store": context.governance_store} if context.governance_store is not None else {})) return try: row = _row(context, status, usage, units) @@ -451,7 +506,7 @@ def _record(context: UsageContext, status: int, usage: Optional[UsageObservation finally: # The settled cost is counted before the in-flight estimate is released. if context.stream_cost is None or context.stream_cost.final_usage is not None or reservation_store.enabled(): - BudgetService.settle(context.reservation) + BudgetService.settle(context.reservation, **({"governance_store": context.governance_store} if context.governance_store is not None else {})) def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], units: Optional[int]) -> dict: @@ -478,6 +533,10 @@ def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], breakdown = CostService.price_buckets(selected or requested, usage, requests=context.units) cost = breakdown["cost_usd"] basis = usage.storage_basis(cost) + from services.tenant_hierarchy import current_tenant, tenant_namespace + from flask import has_request_context + tenant = context.tenant or (current_tenant() if has_request_context() else None) + namespace = tenant_namespace(tenant) if tenant is not None else "" prefix = context.user.get("api_key_prefix") row = { "at": usage_ledger.utc_timestamp(), @@ -495,6 +554,8 @@ def _row(context: UsageContext, status: int, usage: Optional[UsageObservation], "cost_basis": basis, "request_id": context.request_id, } + if namespace: + row["principal"] = credits_admission.owner(tenant) if prompt_cache_cost.enabled() and context.kind in {"chat", "responses", "proxy", "embeddings"}: breakdown = CostService.price_buckets(selected or requested, usage, requests=context.units) row.update({name: breakdown[name] for name in prompt_cache_cost.ROW_FIELDS}) @@ -609,34 +670,59 @@ def fail(error: BaseException) -> None: def admit_item(user: dict, model: str, units: int) -> tuple[Optional[dict], Optional[str]]: - """Allowlist and budget for work run later for an account, such as an asynchronous - batch item. Returns an error for the item, or the budget reservation to settle.""" + """Reserve one background item; its identity owns every admission component.""" + from services.tenant_hierarchy import enabled as organisations_enabled + if organisations_enabled() and not hasattr(g, "tenant_context"): + return {"status": 503, "code": "tenant_storage_unavailable", + "message": "The verified item workspace is unavailable."}, None if not key_controls.model_allowed(user, model): return {"status": 403, "code": "model_not_allowed", "message": f"This API key is not allowed to use {str(model)[:128]}."}, None - if not budgeted(user): + charged = credits_admission.enforcement() != "off" + governance = tenant_governance.workspace_context(user) + if not budgeted(user) and not charged: return None, None - estimate = (reservation_price([model], 0, 0, max(1, units)) if reservation_store.enabled() - else estimate_cost([model], 0, 0, max(1, units))) + estimate = (reservation_price([model], 0, 0, max(1, units)) + if reservation_store.enabled() or governance is not None or charged else + estimate_cost([model], 0, 0, max(1, units))) decision = BudgetService.check_and_reserve(user, estimate) if not decision.allowed: return {"status": decision.status_code, "code": decision.error, "message": decision.message}, None - return None, decision.reservation + from services.tenant_hierarchy import current_tenant + import uuid + identity = decision.reservation or (uuid.uuid4().hex if charged else None) + authority = tenant_governance.store() if tenant_governance.governance_reservation(identity) else None + context = UsageContext("images", [model], None, user, time.perf_counter(), time.time_ns(), + units=max(1, units), reservation=decision.reservation, + tenant=current_tenant(), governance_store=authority) + try: + context.credit_hold = credits_admission.reserve(estimate, scoped_id=identity, context=context.tenant) + except credits_admission.CreditAdmissionError as error: + BudgetService.settle(decision.reservation, before_dispatch=True, governance_store=authority) + return {"status": error.status, "code": error.code, "message": "The item exceeds available credits."}, None + if identity and (charged or authority is not None): + # The Workflow handler admits and records in one request; capture each item, + # rather than sharing its last reservation through g or the parent context. + g.setdefault("managed_item_accounting", {})[identity] = context + return None, identity def record_item(user: dict, *, endpoint: str, requested: str, selected: Optional[str], status: int, units: int, started: float, reservation: Optional[str]) -> None: """Record one item of background work in the ledger and settle its reservation.""" - context = UsageContext(kind="images", models=[requested], provider=None, user=user, started=started, + context = getattr(g, "managed_item_accounting", {}).pop(reservation, None) + context = context or UsageContext(kind="images", models=[requested], provider=None, user=user, started=started, start_ns=time.time_ns(), units=max(1, units), reservation=reservation) context.path, context.selected = endpoint, selected + context.dispatched = True + credits_admission.mark_dispatched(context.credit_hold) _record_row(context, status, units) def _record_row(context: UsageContext, status: int, units: int) -> None: try: row = _row(context, status, None, units) - if reservation_store.enabled() and context.reservation and status < 400 and row["cost_usd"] is not None: + if (reservation_store.enabled() or context.governance_store is not None or context.credit_hold is not None) and status < 400 and row["cost_usd"] is not None: # Flat-price media has measured units on success, even without tokens. row = {**row, "cost_basis": "usage"} row = _record_budget(context, row) @@ -644,4 +730,4 @@ def _record_row(context: UsageContext, status: int, units: int) -> None: except Exception as error: # Accounting must never fail the work it describes. logger.warning("Usage could not be recorded (%s)", type(error).__name__) finally: - BudgetService.settle(context.reservation) + BudgetService.settle(context.reservation, **({"governance_store": context.governance_store} if context.governance_store is not None else {})) diff --git a/services/reservation_store.py b/services/reservation_store.py index 002231bf..88892446 100644 --- a/services/reservation_store.py +++ b/services/reservation_store.py @@ -138,12 +138,26 @@ def transition_fields(state, *, cost_usd=None, basis=None, input_tokens=None, ou def reconcile(store, identity, revision, cost_usd, *, admin, reason, evidence=None, authorized_adjustment=False, transition_id=None): - result = store.transition(identity, revision, "reconciled", cost_usd=cost_usd, + from services import tenant_governance + if tenant_governance.enabled() and tenant_governance.governance_reservation(identity): + try: + result = tenant_governance.store().call("reconcile", id=identity, revision=revision, + cost=None if cost_usd is None else tenant_governance.units(cost_usd), admin=admin, + reason=reason, evidence=evidence, authorized_adjustment=authorized_adjustment, + transition_id=transition_id or uuid.uuid4().hex) + except tenant_governance.GovernanceError as error: + raise ReservationError(error.code, error.status) from None + else: + result = store.transition(identity, revision, "reconciled", cost_usd=cost_usd, admin=admin, reason=reason, evidence=evidence, authorized_adjustment=authorized_adjustment, basis="adjustment" if authorized_adjustment else "provider", transition_id=transition_id or uuid.uuid4().hex, settlement_id=identity) - from services import usage_receipts + from services import credits_admission, usage_receipts + # Replaying the immutable transition also repairs a failed linked settlement. + # Credits operation IDs prevent charging twice if receipt persistence fails. + if result["reservation"]["state"] == "reconciled": + credits_admission.reconcile_reservation(result["reservation"], result["reservation"]["transition_id"]) if result["applied"] and usage_receipts.enabled(): row = result["reservation"] try: diff --git a/services/responses_state.py b/services/responses_state.py index d0d4107d..4c84791b 100644 --- a/services/responses_state.py +++ b/services/responses_state.py @@ -73,7 +73,12 @@ def principal_owner(): principal = user.get("id") or user.get("username") if not isinstance(principal, (str, int)) or not str(principal): raise unavailable() - return hashlib.sha256(canonical([user.get("tenant_id"), str(principal)])).hexdigest() + from services.tenant_hierarchy import tenant_namespace + namespace = tenant_namespace() + identity = [user.get("tenant_id"), str(principal)] + if namespace: + identity.append(namespace) + return hashlib.sha256(canonical(identity)).hexdigest() def _submit(document, stopped, deadline, results): diff --git a/services/saml_federation.py b/services/saml_federation.py index 3ad1065e..e3e3a6e7 100644 --- a/services/saml_federation.py +++ b/services/saml_federation.py @@ -205,6 +205,8 @@ def call(self, operation: str, **values) -> dict: try: body = {"version": 1, "operation": operation, **values} if self._call is None: + if os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip().lower() != "d1": + raise SamlError("saml_storage_unavailable", 503) from services.intelligence_d1_store import request_private_intelligence result = request_private_intelligence(body, endpoint=PRIVATE_ENDPOINT) else: @@ -226,8 +228,12 @@ def lookup_gateway_account(account: str) -> dict | None: def issue_dashboard_session(account: str, context: TenantContext) -> bool: - """An explicit session collaborator is required when no standalone issuer exists.""" - raise SamlError("saml_session_unavailable", 503) + """Use the password-login session issuer after verifying the linked account scope.""" + from services.auth_service import AuthService + principal = account if OPAQUE.fullmatch(account) else "account:" + digest(account) + if type(context) is not TenantContext or context.principal_id != principal: + raise SamlError("saml_identity_denied", 403) + return AuthService.issue_dashboard_session(account) def _active_account(record: object, account: str, now: int) -> bool: diff --git a/services/semantic_generation_cache.py b/services/semantic_generation_cache.py index ec670a62..f8e9ee3d 100644 --- a/services/semantic_generation_cache.py +++ b/services/semantic_generation_cache.py @@ -116,6 +116,8 @@ def cosine(one, two): def partition(principal, path, payload, revisions, policy): + from services.shared_generation_cache import namespace_principal + principal = namespace_principal(principal) messages = [*payload["messages"][:-1], {**payload["messages"][-1], "content": None}] invariant = {**payload, "messages": messages} return {"principal_hash": digest(principal), diff --git a/services/shared_generation_cache.py b/services/shared_generation_cache.py index a9e64cd8..5c2c8b7f 100644 --- a/services/shared_generation_cache.py +++ b/services/shared_generation_cache.py @@ -196,6 +196,13 @@ def put(self, key, body, metadata, *, principal_hash, policy_hash, model): return response.get("stored") is True +def namespace_principal(principal, context=None): + from services.tenant_hierarchy import tenant_namespace + namespace = tenant_namespace(context) + return namespace + "\0" + principal if namespace else principal + + def identity(principal, payload, policy): + principal = namespace_principal(principal) return {"principal_hash": hashlib.sha256(principal.encode()).hexdigest(), "policy_hash": cache_policy.policy_digest(policy), "model": payload["model"]} diff --git a/services/tenant_governance.py b/services/tenant_governance.py index 3b128c75..2dbe1d7f 100644 --- a/services/tenant_governance.py +++ b/services/tenant_governance.py @@ -1,6 +1,7 @@ """Intersected workspace grants and durable multi-level monetary reservations.""" from __future__ import annotations +import hashlib import json import logging import os @@ -115,7 +116,8 @@ def workspace_context(user=None): return None try: context = collaborators().tenant_resolver(user) - if type(context) is not TenantContext or context.principal_id != str(user.get("username") or user.get("id") or ""): + from services.tenant_hierarchy import principal_id + if type(context) is not TenantContext or context.principal_id != principal_id(user.get("username") or user.get("id") or ""): raise GovernanceError("tenant_scope_denied", 403) except GovernanceError: raise @@ -129,7 +131,12 @@ def context_dict(context): def membership_role(principal_id, org_id): - return collaborators().membership_role(principal_id, org_id) + try: + return collaborators().membership_role(principal_id, org_id) + except GovernanceError: + raise + except Exception: + raise GovernanceError() from None class D1GovernanceStore: @@ -261,23 +268,23 @@ def governance_reservation(identity): return isinstance(identity, str) and identity.startswith("tg_") -def dispatch(identity): +def dispatch(identity, *, authority=None): if not governance_reservation(identity): return False - store().call("dispatch", id=identity) + (authority or store()).call("dispatch", id=identity) return True -def complete(identity, row, *, before_dispatch=False, cached=False): +def complete(identity, row, *, before_dispatch=False, cached=False, cancelled=False, authority=None): if not governance_reservation(identity): return False model = row.get("selected_model") model = model if isinstance(model, str) and re.fullmatch(r"[A-Za-z0-9._:/+@\-]{1,256}", model) else None cost = row.get("cost_usd") measured = cost is not None and row.get("cost_basis") == "usage" and not cached - store().call("settle", id=identity, cost=0 if before_dispatch or cached else units(cost) if measured else None, + (authority or store()).call("settle", id=identity, cost=0 if before_dispatch or cached or cancelled else units(cost) if measured else None, provider=model.split(":", 1)[0] if model and ":" in model else None, model=model, - price_basis="released" if before_dispatch else "cache" if cached else row.get("cost_basis"), + price_basis="released" if before_dispatch or cancelled else "cache" if cached else row.get("cost_basis"), before_dispatch=before_dispatch) return True @@ -356,6 +363,8 @@ def call(self, operation, **values): return self._reserve(db, values) if operation in {"settle", "dispatch"}: return self._settle(db, operation, values) + if operation == "reconcile": + return self._reconcile(db, values) if operation == "usage": return self._usage(db, values) raise GovernanceError("invalid_governance_operation", 400) @@ -454,6 +463,49 @@ def _settle(self, db, operation, values): values.get("price_basis"), values["id"])) return {"state": state} + def _reconcile(self, db, values): + from services.credits_ledger import context_owner + cost, revision, transition = values.get("cost"), values.get("revision"), values.get("transition_id") + if (not isinstance(values.get("id"), str) or not re.fullmatch(r"[A-Za-z0-9_:.\-]{1,128}", values["id"]) + or type(values.get("authorized_adjustment", False)) is not bool + or type(revision) is not int or not 0 <= revision < MAX_INTEGER + or not isinstance(transition, str) or not re.fullmatch(r"[a-f0-9]{32}", transition) + or cost is not None and (type(cost) is not int or not 0 <= cost < MAX_INTEGER)): + raise GovernanceError("invalid_governance_operation", 400) + if values.get("admin") is not True: + raise GovernanceError("admin_required", 403) + reason, evidence = values.get("reason"), values.get("evidence") + label = r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}" + adjustment = values.get("authorized_adjustment", False) + if (not isinstance(reason, str) or not re.fullmatch(label, reason) + or adjustment is not True and (not isinstance(evidence, str) or not re.fullmatch(label, evidence)) + or adjustment is True and evidence is not None): + raise GovernanceError("reconciliation_evidence_required", 400) + document = json.dumps([values["id"], revision, cost, reason, evidence, adjustment], separators=(",", ":")) + fingerprint = "reconcile:" + hashlib.sha256(document.encode()).hexdigest() + row = db.execute("SELECT * FROM tenant_governance_reservations WHERE id=?", (values["id"],)).fetchone() + if row is None: + raise GovernanceError("reservation_conflict", 409) + previous = db.execute("SELECT actor,kind FROM tenant_governance_audit WHERE operation_id=?", (transition,)).fetchone() + if previous: + if previous["actor"] != row["id"] or previous["kind"] != fingerprint: + raise GovernanceError("reservation_conflict", 409) + elif row["revision"] != revision or row["state"] not in {"dispatched", "unknown"}: + raise GovernanceError("reservation_conflict", 409) + else: + db.execute("UPDATE tenant_governance_reservations SET state=?,charged=?,revision=revision+1,operation_id=? WHERE id=?", + ("unknown" if cost is None else "settled", cost, transition, row["id"])) + db.execute("INSERT INTO tenant_governance_audit VALUES (?,?,?,?,?,?,?)", + (transition, row["id"], row["org_id"], row["team_id"], revision + 1, fingerprint, + datetime.now(timezone.utc).isoformat())) + context = TenantContext(row["principal_id"], row["org_id"], row["team_id"] or None) + return {"applied": previous is None, "reservation": { + "id": row["id"], "principal": context_owner(context), "day": row["day"], "month": row["month"], + "revision": revision + 1, "state": "unknown" if cost is None else "reconciled", + "cost_usd": None if cost is None else cost / 1_000_000, "transition_id": transition, + "settlement_id": row["id"], "basis": "adjustment" if adjustment else "provider", + "input_tokens": None, "output_tokens": None}} + def _usage(self, db, values): context = TenantContext(**values["context"]) role = values["role"] diff --git a/services/tenant_hierarchy.py b/services/tenant_hierarchy.py index 6e1e3db5..d123f4ee 100644 --- a/services/tenant_hierarchy.py +++ b/services/tenant_hierarchy.py @@ -85,6 +85,11 @@ def _legacy(principal): return legacy_tenant(AuthorityOperation(context, "workspace", 0, "resolve")) +def principal_id(principal): + """The TenantContext principal ID the tenant module uses for an account name.""" + return _legacy(str(principal)).principal_id + + def private_call(document): """Reuse the bounded private transport, with one submission and no replay.""" from services import intelligence_d1_store as transport @@ -134,6 +139,10 @@ def request(self, operation, **values): if os.environ.get("INTELLIGENCE_STORAGE_BACKEND", "").strip() == "d1": return self._remote(operation, values) try: + # Account lookup may initialize schema on the same SQLite file. + # Accounts are revoked rather than removed during provisioning. + if operation == "member_set" and not self.principal_exists(_text(values.get("principal"))): + raise TenantError("principal_not_found", 404) path = storage_path("AUTH_DB_PATH", "auth.sqlite3") if not path.is_file(): raise TenantError("tenant_storage_unavailable", 503) @@ -266,8 +275,6 @@ def _member_write(self, db, v): org_id, principal, data = v["org_id"], _text(v.get("principal")), v.get("data") if not isinstance(data, dict) or not data or set(data) - {"role", "team_id", "status", "bind", "binding_revision"}: raise TenantError("invalid_request") - if not self.principal_exists(principal): - raise TenantError("principal_not_found", 404) row = self._one(db, "SELECT * FROM tenant_memberships WHERE org_id=? AND principal=?", (org_id, principal)) self._cas(row, v.get("revision")) if row is None and db.execute("SELECT count(*) FROM tenant_memberships WHERE org_id=?", (org_id,)).fetchone()[0] >= 1000: @@ -364,3 +371,68 @@ def tenant_context_hook(): def verify_before_key_usage(principal): if enabled() and has_request_context(): g.verified_tenant = resolve_principal(principal, store=current_app.extensions.get("tenant_store")) + + +def _authority_store(store=None): + return store or (current_app.extensions.get("tenant_store") if has_request_context() else None) or TenantStore() + + +def active_membership(principal_id, org_id, *, store=None): + """Read active membership and ancestors; a provisioner cannot manufacture a role.""" + authority = _authority_store(store) + org = authority.request("org_get", org_id=org_id) + if org.get("status") != "active": + return None + members = authority.request("member_list", org_id=org_id)["members"] + member = next((row for row in members if row.get("status") == "active" and + principal_id in {row["principal"], _legacy(row["principal"]).principal_id, + "account:" + hashlib.sha256(row["principal"].encode()).hexdigest()}), None) + if member and member.get("team_id") is not None: + teams = authority.request("team_list", org_id=org_id)["teams"] + if not any(row["id"] == member["team_id"] and row.get("status") == "active" for row in teams): + return None + return member + + +def membership_role(principal_id, org_id, *, store=None): + if not enabled(): + return None + member = active_membership(principal_id, org_id, store=store) + return member["role"] if member else None + + +def tenant_authority(operation, *, store=None): + from services.enterprise_contract import AuthorityDenied + if type(operation) is not AuthorityOperation: + raise TypeError("AuthorityOperation required") + context = operation.context + if context.org_id is None: + return legacy_tenant(operation) + if not enabled(): + raise AuthorityDenied("tenant_scope_denied") + try: + member = active_membership(context.principal_id, context.org_id, store=store) + if not member or member.get("team_id") != context.team_id: + raise AuthorityDenied("tenant_scope_denied") + except TenantError as error: + raise AuthorityDenied(error.code) from None + return context + + +def scim_team_authority(operation, *, scim_store, store=None): + """Verify the bearer-scoped organisation and SCIM members before the atomic team write.""" + from services.enterprise_contract import AuthorityDenied + if type(operation) is not AuthorityOperation or not enabled() or operation.context.org_id is None: + raise AuthorityDenied("tenant_scope_denied") + context = operation.context + try: + org = _authority_store(store).request("org_get", org_id=context.org_id) + if org.get("status") != "active": + raise AuthorityDenied("tenant_scope_denied") + if context.principal_id != "scim": + user = scim_store.get(context.org_id, "Users", operation.scoped_id) + if not user or user.get("userName") != context.principal_id or user.get("active") is not True: + raise AuthorityDenied("tenant_scope_denied") + except TenantError as error: + raise AuthorityDenied(error.code) from None + return context diff --git a/tests/fixtures/governance_reconciliation.json b/tests/fixtures/governance_reconciliation.json new file mode 100644 index 00000000..5e3b18d4 --- /dev/null +++ b/tests/fixtures/governance_reconciliation.json @@ -0,0 +1,82 @@ +[ + { + "revision": 2, + "cost": null, + "transition_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "expected_state": "unknown", + "expected_applied": true + }, + { + "revision": 2, + "cost": null, + "transition_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "expected_state": "unknown", + "expected_applied": false + }, + { + "revision": 3, + "cost": 100000, + "transition_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "admin": false, + "reason": "review", + "evidence": "provider-check", + "error": "admin_required", + "status": 403 + }, + { + "revision": 2, + "cost": 100000, + "transition_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "error": "reservation_conflict", + "status": 409 + }, + { + "revision": 3, + "cost": 100000, + "transition_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "expected_state": "reconciled", + "expected_applied": true + }, + { + "revision": 3, + "cost": 100000, + "transition_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "expected_state": "reconciled", + "expected_applied": false + }, + { + "revision": 3, + "cost": 200000, + "transition_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "error": "reservation_conflict", + "status": 409 + }, + { + "revision": 4, + "cost": 100000, + "transition_id": "cccccccccccccccccccccccccccccccc", + "admin": true, + "reason": "review", + "evidence": "provider-check", + "error": "reservation_conflict", + "status": 409 + } +] diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 5d5d1bd1..48e75ce1 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -295,6 +295,7 @@ "worker/control-state-d1.mjs", "worker/control-users-d1.mjs", "worker/cors-policy.mjs", + "worker/credits-admission.mjs", "worker/credits-d1.mjs", "worker/d1-schema.mjs", "worker/enterprise-contract.mjs", diff --git a/tests/test_cascades.py b/tests/test_cascades.py index 5ba9c1a5..97a02887 100644 --- a/tests/test_cascades.py +++ b/tests/test_cascades.py @@ -229,8 +229,8 @@ def test_secret_firewall_failure_is_not_swallowed(context): def test_budget_denial_keeps_best(context, monkeypatch): g.authenticated_user = {'username': 'synthetic', 'daily_budget_usd': 1} - from services import accounted_dispatch as dispatch_module - monkeypatch.setattr(dispatch_module, 'budgeted', lambda user: True) + from services import request_accounting as accounting_module + monkeypatch.setattr(accounting_module, 'budgeted', lambda user: True) decisions = iter([BudgetDecision(True, reservation='synthetic-reservation'), BudgetDecision(False, status_code=402, error='budget_exceeded', message='Synthetic budget denial')]) monkeypatch.setattr(BudgetService, 'check_and_reserve', lambda *args: next(decisions)) response, calls = run(CONFIG, [answer('partial', 'length')]) diff --git a/tests/test_credits_ledger.py b/tests/test_credits_ledger.py index 4d3cb02f..f7dabde5 100644 --- a/tests/test_credits_ledger.py +++ b/tests/test_credits_ledger.py @@ -34,12 +34,13 @@ def test_shared_vectors(store): assert error.value.status == 400 continue values = vector['request'] + apply = store.append_payment if vector.get('operation') == 'payment_append' else store.append if vector.get('error'): with pytest.raises(ledger.CreditsError) as error: - store.append(**values) + apply(**values) assert error.value.code == vector['error'] else: - result = store.append(**values) + result = apply(**values) assert result['revision'] == vector['revision'] if 'summary' in vector: summary = store.read(values['owner']) diff --git a/tests/test_credits_ledger_worker.mjs b/tests/test_credits_ledger_worker.mjs index e99b2f01..b4ca9eb9 100644 --- a/tests/test_credits_ledger_worker.mjs +++ b/tests/test_credits_ledger_worker.mjs @@ -3,7 +3,7 @@ import test from 'node:test'; import { readFileSync } from 'node:fs'; import { DatabaseSync } from 'node:sqlite'; import { spawnSync } from 'node:child_process'; -import { handleCreditsRequest, appendCredit, readCredits, registerCreditAuthority, creditsEnabled, contextOwner } from '../worker/credits-d1.mjs'; +import { handleCreditsRequest, appendCredit, appendPaymentCredit, readCredits, registerCreditAuthority, creditsEnabled, creditsEnforcement, contextOwner } from '../worker/credits-d1.mjs'; import { AuthorityOperation, TenantContext, callAuthority, registerEnterpriseAdapters } from '../worker/enterprise-contract.mjs'; const migration = readFileSync(new URL('../intelligence-migrations/0037_credits_ledger.sql', import.meta.url), 'utf8'); @@ -50,7 +50,7 @@ test('shared integer, insufficient and duplicate operation vectors', async t => const { call } = database(t); for (const vector of vectors) { const request = vector.request ?? entry('credit', vector.invalid_amount, 'bad', 0); - const result = await call({ operation: 'append', ...request }); + const result = await call({ operation: vector.operation ?? 'append', ...request }); if ('invalid_amount' in vector) assert.equal(result.status, 400); else if (vector.error) assert.equal(result.error.code, vector.error); else assert.equal(result.entry.revision, vector.revision); @@ -216,11 +216,25 @@ test('migration rehearsal, incomplete columns, audit and bounded pagination', as test('Python and D1 return identical ledger vectors', async t => { const { db } = database(t); - for (const vector of vectors.filter(vector => vector.request && !vector.error)) await appendCredit(db, vector.request); - const code = `import os,sys,json,tempfile\nfrom pathlib import Path\nsys.path.insert(0,os.getcwd())\nfrom services.credits_ledger import SqlCreditsLedger\ntext=Path('docs/credits-ledger.md').read_text()\nvectors=json.loads(text.split('\x60\x60\x60json\\n',1)[1].split('\x60\x60\x60',1)[0])\nwith tempfile.TemporaryDirectory() as d:\n store=SqlCreditsLedger(Path(d)/'credits.sqlite3',initialize=True)\n for v in vectors:\n if 'request' in v and not v.get('error'): store.append(**v['request'])\n print(json.dumps(store.read('alice')))\n`; + for (const vector of vectors.filter(vector => vector.request && !vector.error)) { + await (vector.operation === 'payment_append' ? appendPaymentCredit : appendCredit)(db, vector.request); + } + const code = `import os,sys,json,tempfile\nfrom pathlib import Path\nsys.path.insert(0,os.getcwd())\nfrom services.credits_ledger import SqlCreditsLedger\ntext=Path('docs/credits-ledger.md').read_text()\nvectors=json.loads(text.split('\x60\x60\x60json\\n',1)[1].split('\x60\x60\x60',1)[0])\nwith tempfile.TemporaryDirectory() as d:\n store=SqlCreditsLedger(Path(d)/'credits.sqlite3',initialize=True)\n for v in vectors:\n if 'request' in v and not v.get('error'): (store.append_payment if v.get('operation') == 'payment_append' else store.append)(**v['request'])\n print(json.dumps(store.read('alice')))\n`; const python = spawnSync('/home/mysterious/storage/github/MultiLLM-Proxy/.venv/bin/python', ['-I', '-c', code], { cwd: new URL('..', import.meta.url), encoding: 'utf8', timeout: 10000, }); assert.equal(python.status, 0, python.stderr); assert.deepEqual(await readCredits(db, 'alice'), JSON.parse(python.stdout)); }); + + test('enforcement parser is feature-gated and warns once without values', () => { + const warnings = []; + const warn = value => warnings.push(value); + for (const value of ['', 'off', 'funded', 'all', ' ALL ']) { + assert.equal(creditsEnforcement({ CREDITS_ENABLED: 'true', CREDITS_ENFORCEMENT: value }, warn), value.trim().toLowerCase() || 'off'); + } + assert.equal(creditsEnforcement({ CREDITS_ENFORCEMENT: 'all' }, warn), 'off'); + for (let n = 0; n < 2; n++) assert.equal(creditsEnforcement({ CREDITS_ENABLED: 'true', CREDITS_ENFORCEMENT: 'private-invalid' }, warn), 'off'); + assert.equal(warnings.length, 1); + assert.ok(!warnings[0].includes('private-invalid')); +}); diff --git a/tests/test_image_quality.py b/tests/test_image_quality.py index 01160a98..9236684f 100644 --- a/tests/test_image_quality.py +++ b/tests/test_image_quality.py @@ -250,8 +250,7 @@ def test_judge_obeys_the_key_model_allowlist(self): def test_budget_stop_keeps_the_first_image(self): decisions = iter([BudgetDecision(True), BudgetDecision(True), BudgetDecision(True), BudgetDecision(False, error="budget_exceeded", status_code=429, message="Synthetic budget spent")]) - with patch("services.accounted_dispatch.budgeted", return_value=True), \ - patch("services.request_accounting.budgeted", return_value=True), \ + with patch("services.request_accounting.budgeted", return_value=True), \ patch.object(BudgetService, "check_and_reserve", side_effect=lambda *args: next(decisions)): response, generations, judges, _ = self.generate([3]) self.assertEqual(response.status_code, 200) @@ -292,8 +291,7 @@ def ordered(*args): return result decisions = iter([BudgetDecision(True)] * 3 + [ BudgetDecision(False, error="budget_exceeded", status_code=429, message="Synthetic budget spent")]) - with patch("services.accounted_dispatch.budgeted", return_value=True), \ - patch("services.request_accounting.budgeted", return_value=True), \ + with patch("services.request_accounting.budgeted", return_value=True), \ patch.object(qa_routes, "_one_image", side_effect=ordered), \ patch.object(BudgetService, "check_and_reserve", side_effect=lambda *args: next(decisions)): response, generations, judges, _ = self.generate([9], n=2) @@ -308,7 +306,7 @@ def test_mixed_batch_accounts_both_paths_and_reserves_non_qa_image_units(self): body = {"defaults": {"model": MODEL}, "items": [ {"prompt": "A square", "quality_check": {"judge_model": JUDGE}}, {"prompt": "Two circles", "n": 2}]} - with patch("services.accounted_dispatch.budgeted", return_value=True), \ + with patch("services.request_accounting.budgeted", return_value=True), \ patch.object(BudgetService, "check_and_reserve", return_value=BudgetDecision(True)) as reserve: response, generations, judges, _ = self.generate([9], path="/v1/images/batch", body=body) self.assertEqual(response.status_code, 200) @@ -355,8 +353,7 @@ def test_retry_judge_error_keeps_previously_graded_best(self): def test_retry_judge_budget_stop_is_reported(self): decisions = iter([BudgetDecision(True)] * 4 + [ BudgetDecision(False, error="budget_exceeded", status_code=429, message="Synthetic budget spent")]) - with patch("services.accounted_dispatch.budgeted", return_value=True), \ - patch("services.request_accounting.budgeted", return_value=True), \ + with patch("services.request_accounting.budgeted", return_value=True), \ patch.object(BudgetService, "check_and_reserve", side_effect=lambda *args: next(decisions)): response, generations, judges, _ = self.generate([3]) self.assertEqual(response.status_code, 200) diff --git a/tests/test_image_quality_parallel_r2.py b/tests/test_image_quality_parallel_r2.py index 4e2b1ff4..94c7344a 100644 --- a/tests/test_image_quality_parallel_r2.py +++ b/tests/test_image_quality_parallel_r2.py @@ -91,8 +91,7 @@ def raw(*args, **kwargs): def judge(*args, **kwargs): judging.wait(timeout=5) return Response(json.dumps(qa_tests.completion(qa_tests.grade(3))), content_type="application/json") - with patch("services.accounted_dispatch.budgeted", return_value=True), \ - patch("services.request_accounting.budgeted", return_value=True), \ + with patch("services.request_accounting.budgeted", return_value=True), \ patch.object(BudgetService, "check_and_reserve", side_effect=reserve), \ patch("routes.unified_images.dispatch_image_generation_raw", side_effect=raw) as generate, \ patch("routes.unified._dispatch_unified_chat_candidate", side_effect=judge): diff --git a/tests/test_saml_federation.py b/tests/test_saml_federation.py index e2a9a856..80150dab 100644 --- a/tests/test_saml_federation.py +++ b/tests/test_saml_federation.py @@ -171,7 +171,7 @@ def test_valid_assertion_uses_only_existing_account_and_bound_authority(register authority = Mock(side_effect=lambda assertion, operation: operation.context) service.identity_authority = authority response = callback(client, signer, claims, method) - assert response.status_code == 302 and response.location == "/" + assert response.status_code == 302 and urlsplit(response.location).path == "/" issue.assert_called_once_with("alice", TenantContext("alice", "org:one", "team:one", 2)) assertion, operation = authority.call_args.args assert assertion.verified and operation.context.org_id == "org:one" @@ -234,7 +234,7 @@ def test_missing_storage_and_session_fail_closed(registered, signer): assert b"private storage" not in response.data issue.assert_not_called() store.available = True - service.session_issuer = saml.issue_dashboard_session + service.session_issuer = Mock(side_effect=saml.SamlError("saml_session_unavailable", 503)) response = callback(client, signer, claims) assert response.status_code == 503 and response.json["error"]["code"] == "saml_session_unavailable" @@ -386,22 +386,19 @@ def issue(account, context): session["user"] = {"username": account, "is_admin": accounts[account]["is_admin"], "scopes": accounts[account]["scopes"], "api_key_prefix": "synthetic", "session_id": "synthetic-session"} return True - service = saml.SamlFederation(saml.load_config(env), saml.SamlStore(store), session_issuer=issue, account_lookup=accounts.get, clock=lambda: NOW) - app.extensions["saml_federation"] = service - register_saml_federation_routes(app, app.extensions["csrf"]) + service = app.extensions["saml_federation"] + service.config, service.store = saml.load_config(env), saml.SamlStore(store) + service.session_issuer, service.account_lookup, service.clock = issue, accounts.get, lambda: NOW + service.identity_authority = saml.stored_link_authority link(service, org_id=None, team_id=None, grants_revision=0) client = app.test_client() - # Exercise the existing guard first, then the required static allowlist integration. - blocked = client.get("/auth/saml/login") - assert blocked.status_code == 302 and urlsplit(blocked.location).path == "/login" guard = next(hook for hook in app.before_request_funcs[None] if hook.__name__ == "handle_redirects") - monkeypatch.setitem(guard.__globals__, "PRODUCT_PUBLIC_ENDPOINTS", guard.__globals__["PRODUCT_PUBLIC_ENDPOINTS"] | SAML_PUBLIC_ENDPOINTS) headers = next(hook for hook in app.after_request_funcs[None] if hook.__name__ == "add_response_headers") for auth in {guard.__globals__["AuthService"], headers.__globals__["AuthService"]}: monkeypatch.setattr(auth, "_load_user_by_username", accounts.get) claims = start(client) response = callback(client, signer, claims, "POST") - assert response.status_code == 302 + assert response.status_code == 302 and urlsplit(response.location).path == "/" with client.session_transaction() as saved: assert saved["authenticated"] is True and saved["user"]["is_admin"] is False assert saved["user"]["scopes"] == ["chat"] diff --git a/tests/test_tenant_governance.py b/tests/test_tenant_governance.py index 45ba8890..05c95e0e 100644 --- a/tests/test_tenant_governance.py +++ b/tests/test_tenant_governance.py @@ -362,3 +362,28 @@ def test_disabled_and_legacy_usage_are_identical(self): governance.register_governance_collaborators(app=self.app, store=self.store) legacy = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) assert disabled.status_code == legacy.status_code == 200 and disabled.data == legacy.data + + +def test_shared_reconciliation_vectors_keep_unknown_holds_and_settle_once(setup): + import json + app, store = setup + identity = "tg_reconciliation" + store.call("reserve", id=identity, context=governance.context_dict(TENANT), amount=400000, + day="2026-10-09", month="2026-10", key_daily=1000000, key_monthly=None, + base_day=0, base_month=0) + store.call("dispatch", id=identity) + store.call("settle", id=identity, cost=None) + vectors = json.loads(Path("tests/fixtures/governance_reconciliation.json").read_text()) + for vector in vectors: + values = {key: value for key, value in vector.items() if not key.startswith("expected_") and key not in {"error", "status"}} + if "error" in vector: + with pytest.raises(governance.GovernanceError) as failure: + store.call("reconcile", id=identity, **values) + assert (failure.value.code, failure.value.status) == (vector["error"], vector["status"]) + else: + result = store.call("reconcile", id=identity, **values) + assert result["applied"] is vector["expected_applied"] + assert result["reservation"]["state"] == vector["expected_state"] + totals = store.call("usage", context=governance.context_dict(TENANT), role="admin") + assert totals["holds_micro_usd"] == (400000 if vector["cost"] is None else 0) + assert totals["spent_micro_usd"] == (0 if vector["cost"] is None else 100000) diff --git a/tests/test_tenant_governance_worker.mjs b/tests/test_tenant_governance_worker.mjs index ef78afd5..a34ff9fc 100644 --- a/tests/test_tenant_governance_worker.mjs +++ b/tests/test_tenant_governance_worker.mjs @@ -212,3 +212,25 @@ test("native flat cache events release holds and ancestor errors retain the leve await assert.rejects(create().admit({ model: "openai:gpt", key_allowed: true, amount: 500_000 }), error => error.status === 429 && error.details.level === "organisation"); }); + + +test("shared reconciliation vectors keep unknown holds and settle once", async t => { + const { call } = database(t); + const id = "tg_reconciliation"; + await call(reserve("reconciliation")); + await call({ operation: "dispatch", id }); + await call({ operation: "settle", id, cost: null }); + const vectors = JSON.parse(readFileSync(new URL("./fixtures/governance_reconciliation.json", import.meta.url), "utf8")); + for (const vector of vectors) { + const values = Object.fromEntries(Object.entries(vector).filter(([key]) => !key.startsWith("expected_") && !["error", "status"].includes(key))); + const result = await call({ operation: "reconcile", id, ...values }); + if (vector.error) { assert.equal(result.status, vector.status); assert.equal(result.error.code, vector.error); } + else { + assert.equal(result.status, 200); assert.equal(result.applied, vector.expected_applied); + assert.equal(result.reservation.state, vector.expected_state); + const totals = await call({ operation: "usage", context, role: "admin" }); + assert.equal(totals.holds_micro_usd, vector.cost === null ? 400000 : 0); + assert.equal(totals.spent_micro_usd, vector.cost === null ? 0 : 100000); + } + } +}); diff --git a/tests/test_tenant_hierarchy.py b/tests/test_tenant_hierarchy.py index 535ee815..e27cb6b0 100644 --- a/tests/test_tenant_hierarchy.py +++ b/tests/test_tenant_hierarchy.py @@ -351,3 +351,41 @@ def test_bootstrap_tenant_check_follows_environment_key_verification(monkeypatch assert auth.verify_api_key("synthetic-bootstrap") == {"username": "operator"} tenant.assert_called_once_with("operator") verified.assert_called_once_with("operator", "synthetic-bootstrap", None) + + + +def test_real_account_lookup_precedes_membership_write_transaction(store, monkeypatch): + auth = importlib.import_module("services.auth_service").AuthService + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "") + with auth._connect() as db: + auth._ensure_users_schema(db) + db.execute("INSERT INTO users (username,api_key_hash,api_key_prefix,scopes,is_admin,created_at) VALUES (?,?,?,?,?,?)", + ("alice", "synthetic-hash", "synthetic", '["chat"]', 0, "2026-10-09")) + real_store = th.TenantStore() + org, team = hierarchy(real_store) + assert member(real_store, org, team)["principal"] == "alice" + + +def test_account_names_outside_the_opaque_format_keep_their_workspace(monkeypatch): + from services import media_signing, tenant_governance + from services.budget_service import principal_owner + name = "alice@example.com" + legacy = th._legacy(name) + assert th.principal_id("alice") == "alice" + assert th.principal_id(name) == legacy.principal_id and legacy.principal_id.startswith("principal:") + workspace = th.TenantContext(legacy.principal_id, "org1", "team1") + monkeypatch.setenv("ORGANISATIONS_ENABLED", "true") + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + monkeypatch.setenv("MEDIA_SIGNING_SECRET", "synthetic-signing-secret") + app = Flask(__name__) + tenant_governance.register_governance_collaborators(tenant_resolver=lambda user: workspace, app=app) + with app.test_request_context("/v1/chat/completions"): + g.tenant_context = workspace + user = {"username": name} + assert tenant_governance.workspace_context(user) == workspace + assert principal_owner(user) != name + token = media_signing.issue_principal("batch", "job1", name, 60) + claims = media_signing.read_principal(token, "batch") + with app.test_request_context("/internal/batch"): + assert media_signing.bind_principal_tenant(claims, {"username": name}) == workspace + assert g.tenant_context == workspace diff --git a/tests/test_wave7_managed.py b/tests/test_wave7_managed.py new file mode 100644 index 00000000..d01977fc --- /dev/null +++ b/tests/test_wave7_managed.py @@ -0,0 +1,569 @@ +"""Request admission and workspace isolation with synthetic storage and upstreams.""" +import hashlib +import importlib +import json +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask, g, jsonify + +from tests.test_wave5_managed import gateway as base_gateway, body, post +from tests.test_protocol_routes import RESPONSES_BODY, CHAT_STREAM, json_upstream, sse_upstream + +FLAGS = ("ORGANISATIONS_ENABLED", "TENANT_GOVERNANCE_ENABLED", "SAML_ENABLED", + "SCIM_ENABLED", "CREDITS_ENABLED", "PAYMENTS_ENABLED") + + +@pytest.fixture(autouse=True) +def defaults(monkeypatch): + for flag in FLAGS: + monkeypatch.setenv(flag, "false") + monkeypatch.setenv("CREDITS_ENFORCEMENT", "off") + monkeypatch.setenv("AUTH_STORAGE_BACKEND", "") + for flag in ("HEDGED_REQUESTS_ENABLED", "STREAM_COST_BREAKER_ENABLED", "USAGE_RECEIPTS_ENABLED"): + monkeypatch.setenv(flag, "false") + monkeypatch.setenv("CONTEXT_CANARY_MODE", "off") + monkeypatch.setenv("LATENCY_SLO_MODE", "off") + + +@pytest.fixture +def gateway(base_gateway): + base_gateway.headers["X-Request-ID"] = "managed-fixed" + return base_gateway + + +@pytest.fixture +def credits(tmp_path, monkeypatch): + admission = importlib.import_module("services.credits_admission") + ledger = importlib.import_module("services.credits_ledger") + contracts = importlib.import_module("services.enterprise_contract") + path = tmp_path / "credits.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0037_credits_ledger.sql").read_text()) + store = ledger.SqlCreditsLedger(path) + monkeypatch.setenv("CREDITS_ENABLED", "true") + monkeypatch.setenv("CREDITS_ENFORCEMENT", "all") + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", '{"mimo:mimo-v2.5":{"input":1,"output":1}}') + monkeypatch.setattr(admission.credits_ledger, "open_store", lambda: store) + app = Flask(__name__) + app.config["TESTING"] = True + authority = admission.register_credits_admission(app) + app.extensions["enterprise_adapters"] = contracts.register_enterprise_adapters(credit=authority) + return app, store, admission, contracts + + +def fund(store, owner="alice", amount=1000): + store.append(owner, "credit", amount, "fund", 0) + + +@pytest.mark.parametrize("mode", ["funded", "all"]) +def test_reserve_commit_and_unknown_hold(credits, monkeypatch, mode): + app, store, admission, contracts = credits + monkeypatch.setenv("CREDITS_ENFORCEMENT", mode) + fund(store) + with app.test_request_context("/v1/chat/completions"): + g.authenticated_user = {"username": "alice"} + hold = admission.reserve(0.0001, scoped_id="attempt1") + assert store.read("alice")["held_microusd"] == 100 + admission.mark_dispatched(hold) + admission.complete(hold, {"cost_usd": 0.000025, "cost_basis": "usage"}) + admission.complete(hold, {"cost_usd": 0.000025, "cost_basis": "usage"}) + assert store.read("alice")["balance_microusd"] == 975 + unknown = admission.reserve(0.0001, scoped_id="attempt2") + admission.mark_dispatched(unknown) + admission.complete(unknown, {"cost_usd": None, "cost_basis": None}) + admission.complete(unknown, {}) + rows = store.read("alice") + assert rows["held_microusd"] == 100 + assert [entry["kind"] for entry in rows["entries"]] == ["credit", "reserve", "commit", "reserve", "reserve"] + + +def test_funded_unfunded_and_enforcement_off_never_charge(credits, monkeypatch): + app, store, admission, _ = credits + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + monkeypatch.setenv("CREDITS_ENFORCEMENT", "funded") + assert admission.reserve(None, scoped_id="unfunded") is None + monkeypatch.setenv("CREDITS_ENFORCEMENT", "off") + with patch.object(store, "read", side_effect=AssertionError("enforcement off read credits")): + assert admission.reserve(None, scoped_id="disabled") is None + + +@pytest.mark.parametrize("estimate,code,status", [(None, "credits_unpriced", 503), (0, "credits_unpriced", 503), + (0.001001, "credits_insufficient", 402)]) +def test_credit_denials_are_predispatch(credits, estimate, code, status): + app, store, admission, _ = credits + fund(store) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + with pytest.raises(admission.CreditAdmissionError) as failure: + admission.reserve(estimate, scoped_id="denied") + assert failure.value.code == code and failure.value.status == status + assert store.read("alice")["held_microusd"] == 0 + + +def test_revision_retry_keeps_operation_identity(credits, monkeypatch): + app, store, admission, contracts = credits + fund(store) + original = store.append + attempts = [] + def append(*args, **kwargs): + attempts.append((args, kwargs)) + if len(attempts) == 1: + original("alice", "adjust", 1, "concurrent", 1, actor="admin", reason="allocation") + return original(*args, **kwargs) + monkeypatch.setattr(store, "append", append) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + hold = admission.reserve(0.0001, scoped_id="same-attempt") + admission.release(hold) + admission.release(hold) + assert len(attempts) == 3 + assert attempts[0][0][3] == attempts[1][0][3] + assert attempts[0][0][4] == 1 and attempts[1][0][4] == 2 + assert attempts[0][1]["scoped_id"] == attempts[1][1]["scoped_id"] == "same-attempt" + assert store.read("alice")["held_microusd"] == 0 + assert store.read("alice")["entries"][-1]["kind"] == "release" + + +@pytest.mark.parametrize("path,payload,result", [ + ("/v1/chat/completions", body(), None), + ("/v1/messages", {"model": "mimo:mimo-v2.5", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 10}, None), + ("/v1/responses", {"model": "opencode:grok-4.6", "input": "hi"}, RESPONSES_BODY), + ("/v1/chat/completions", body(model="auto:managed-test"), None), + ("/v1/chat/completions", body(model="opencode:kimi-k2.6", stream=True), "stream"), +]) +def test_flags_off_keep_registered_request_bytes(gateway, monkeypatch, path, payload, result): + monkeypatch.setattr("time.perf_counter", lambda: 1000) + importlib.import_module("services.auto_route_service").AutoRouteService.save_route( + "auto:managed-test", ["mimo:mimo-v2.5"], gateway.app.config["API_BASE_URLS"]) + def reply(): + return sse_upstream(CHAT_STREAM) if result == "stream" else json_upstream(result) if result else gateway._chat_response() + first, send = post(gateway, payload, path=path, reply=reply()) + submitted = send.call_args.kwargs["data"] + for flag in (*FLAGS, "CREDITS_ENFORCEMENT"): + monkeypatch.delenv(flag, raising=False) + second, send = post(gateway, payload, path=path, reply=reply()) + assert first.status_code == second.status_code == 200 + assert first.data == second.data and submitted == send.call_args.kwargs["data"] + assert list(first.headers) == list(second.headers) + + +def test_legacy_namespace_bytes_and_workspace_separation(monkeypatch): + contracts = importlib.import_module("services.enterprise_contract") + shared = importlib.import_module("services.shared_generation_cache") + semantic = importlib.import_module("services.semantic_generation_cache") + idempotency = importlib.import_module("services.idempotency_store") + responses = importlib.import_module("services.responses_state") + pages = importlib.import_module("services.context_pages") + accounting = importlib.import_module("services.request_accounting") + app = Flask(__name__) + user = {"username": "alice", "id": 7} + policy = importlib.import_module("services.cache_policy").CachePolicy("openai:gpt", "1", "1", "1", "1") + payload = body() + with app.test_request_context("/v1/chat/completions", headers={"Authorization": "Bearer synthetic", "Session-Id": "session"}, json=payload): + g.authenticated_user = user + principal = "alice\0key-hash" + scope, digest = idempotency.request_fingerprint("alice", "POST", "/v1/chat/completions", "id", payload, "1") + assert scope == hashlib.sha256(idempotency._canonical(["alice", "POST", "/v1/chat/completions", "id"])).hexdigest() + assert digest == hashlib.sha256(idempotency._canonical([payload, "1"])).hexdigest() + assert responses.principal_owner() == hashlib.sha256(responses.canonical([None, "7"])).hexdigest() + legacy = shared.identity(principal, payload, policy) + assert legacy["principal_hash"] == hashlib.sha256(principal.encode()).hexdigest() + legacy_semantic = semantic.partition(principal, "/v1/chat/completions", payload, {}, {}) + legacy_pages = pages.managed_authority()["scope"].principal + context = accounting.UsageContext("chat", [payload["model"]], None, user, 1000, 1) + row = accounting._row(context, 200, None, None) + assert row["principal"] == "alice" and "workspace" not in row + snapshots = [] + for org in ("one", "two"): + g.tenant_context = contracts.TenantContext("alice", org, "team") + snapshots.append((shared.identity(principal, payload, policy), + semantic.partition(principal, "/v1/chat/completions", payload, {}, {}), + idempotency.request_fingerprint("alice", "POST", "/v1/chat/completions", "id", payload, "1"), + responses.principal_owner(), pages.managed_authority()["scope"])) + assert all(a != b for a, b in zip(*snapshots)) + assert snapshots[0][0] != legacy and snapshots[0][1] != legacy_semantic + assert snapshots[0][4].principal != legacy_pages + + +@pytest.mark.parametrize("mode", ["off", "funded", "all"]) +def test_registered_charged_generation(gateway, credits, monkeypatch, mode): + _, store, admission, _ = credits + gateway.app.extensions.update(credits[0].extensions) + monkeypatch.setenv("CREDITS_ENFORCEMENT", mode) + monkeypatch.setenv("RESPONSE_CACHE_ENABLED", "false") + fund(store, "admin", 10000) + response, send = post(gateway, reply=json_upstream({"model": "mimo:mimo-v2.5", + "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 20}})) + assert response.status_code == 200, response.data + assert send.call_count == 1 + rows = store.read("admin") + assert rows["held_microusd"] == 0 + assert [entry["kind"] for entry in rows["entries"]] == (["credit"] if mode == "off" else ["credit", "reserve", "commit"]) + + +def test_governance_denial_precedes_credits_and_lazy_completion(credits, tmp_path, monkeypatch): + app, store, admission, contracts = credits + governance = importlib.import_module("services.tenant_governance") + accounting = importlib.import_module("services.request_accounting") + budget = importlib.import_module("services.budget_service") + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + authority = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path)) + tenant = contracts.TenantContext("alice", "org", "team") + governance.register_governance_collaborators(app=app, store=authority, tenant_resolver=lambda user: tenant) + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + authority.call("put", org_id="org", team_id=None, actor="alice", revision=0, + policy={"models": None, "tools": None, "daily": 0, "monthly": None}) + fund(store, admission.owner(tenant), 10000) + budget.BudgetService.reset() + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice"} + g.tenant_context = tenant + with patch.object(store, "append", wraps=store.append) as append: + denied = accounting.begin() + assert denied.status_code == 429 and denied.json["error"] == "tenant_budget_exceeded" + append.assert_not_called() + authority.call("put", org_id="org", team_id=None, actor="alice", revision=1, + policy={"models": None, "tools": None, "daily": 10000, "monthly": None}) + assert accounting.begin() is None + context = g.usage_context + accounting.mark_dispatched() + accounting._record_budget(context, {"cost_usd": 0.000025, "cost_basis": "usage", "status": 200, + "selected_model": "mimo:mimo-v2.5"}) + totals = authority.call("usage", context=governance.context_dict(tenant), role="admin") + assert totals["holds_micro_usd"] == 0 and totals["spent_micro_usd"] == 25 + assert store.read(admission.owner(tenant))["balance_microusd"] == 9975 + budget.BudgetService.reset() + + +def test_unknown_durable_reconciliation_charges_once(credits, tmp_path, monkeypatch): + app, store, admission, _ = credits + reservations = importlib.import_module("services.reservation_store") + durable = reservations.SqlReservationStore(tmp_path / "usage.sqlite3") + identity = "a" * 32 + transition = "b" * 32 + durable.reserve(identity, "alice", 0.0001, 1, None, 0, 0) + fund(store) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + hold = admission.reserve(0.0001, scoped_id=identity) + admission.mark_dispatched(hold) + admission.complete(hold, {}) + durable.transition(identity, 0, "dispatched", transition_id="c" * 32) + durable.transition(identity, 1, "unknown", transition_id="d" * 32) + for _ in range(2): + reservations.reconcile(durable, identity, 2, 0.000025, admin=True, reason="provider_usage", + evidence="receipt1", transition_id=transition) + assert store.read("alice")["balance_microusd"] == 975 + assert store.read("alice")["held_microusd"] == 0 + assert len([entry for entry in store.read("alice")["entries"] if entry["kind"] == "commit"]) == 1 + + +def test_isolated_attempts_bind_distinct_scopes(credits): + app, store, admission, _ = credits + managed = importlib.import_module("services.managed_dispatch") + fund(store) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + identities = [] + for _ in range(2): + with managed.isolated_managed_attempt(None): + hold = admission.reserve(0.0001, scoped_id=managed.current_attempt_id()) + identities.append(hold.scoped_id) + admission.release(hold) + assert managed.current_attempt_id() is None + assert identities[0] != identities[1] and store.read("alice")["held_microusd"] == 0 + + +def test_pre_dispatch_rejection_releases_request_once(credits): + app, store, admission, _ = credits + accounting = importlib.import_module("services.request_accounting") + fund(store) + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice"} + assert accounting.begin() is None + assert store.read("alice")["held_microusd"] > 0 + accounting.release_before_dispatch() + accounting.release_before_dispatch() + assert store.read("alice")["held_microusd"] == 0 + assert [entry["kind"] for entry in store.read("alice")["entries"]] == ["credit", "reserve", "release"] + + +def test_conflict_is_not_retried_and_settlement_retains_hold(credits, monkeypatch): + app, store, admission, _ = credits + ledger = importlib.import_module("services.credits_ledger") + fund(store) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + hold = admission.reserve(0.0001, scoped_id="attempt") + admission.mark_dispatched(hold) + with patch.object(store, "append", side_effect=ledger.CreditsError("credits_conflict", 409)) as append: + with pytest.raises(admission.CreditAdmissionError) as error: + admission.complete(hold, {"cost_usd": 0.000025, "cost_basis": "usage"}) + assert error.value.code == "credits_unavailable" and append.call_count == 1 + assert not hold.terminal and store.read("alice")["held_microusd"] == 100 + + +@pytest.mark.parametrize("workspace", [False, True]) +def test_hedged_attempts_own_separate_credit_settlement(credits, tmp_path, monkeypatch, workspace): + import threading + from flask import Response + app, store, admission, contracts = credits + accounting = importlib.import_module("services.request_accounting") + hedges = importlib.import_module("services.hedged_requests") + budget = importlib.import_module("services.budget_service") + reservations = importlib.import_module("services.reservation_store") + governance = importlib.import_module("services.tenant_governance") + tenant = contracts.TenantContext("alice", "org", "team") if workspace else contracts.TenantContext("alice") + authority = None + if workspace: + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + authority = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path)) + governance.register_governance_collaborators(app=app, store=authority, tenant_resolver=lambda user: tenant) + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + fund(store, admission.owner(tenant)) + models = ["mimo:mimo-v2.5", "opencode:second"] + monkeypatch.setenv("MODEL_PRICING_USD_PER_MILLION", json.dumps({model: {"input": 1, "output": 1} for model in models})) + budget.BudgetService.reset() + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice", "daily_budget_usd": 1} + g.tenant_context = tenant + assert accounting.begin() is None + outer = g.usage_context + # The legacy in-memory budget hold has the same undispatched bounds. + with patch.object(reservations, "get_store") as durable: + durable.return_value.get.return_value = {"state": "reserved", "estimate_usd": outer.estimate} + holds = hedges._prepare_holds(models, body()) + attempts = [hedges._Attempt(model, i, holds[i], outer, 1010, lambda: 1000, threading.Condition()) + for i, model in enumerate(models)] + assert attempts[0].context.credit_hold.scoped_id != attempts[1].context.credit_hold.scoped_id + for attempt in attempts: + attempt._dispatch_started() + attempt.response = Response(status=200) + attempts[0].accepted = True + attempts[0].usage = accounting.UsageObservation(10, 20, "provider", ("provider",)) + attempts[1].cancel() + for attempt in attempts: + attempt._settle() + rows = store.read(admission.owner(tenant)) + assert rows["held_microusd"] == 0 and rows["balance_microusd"] == 970 + assert [entry["kind"] for entry in rows["entries"]] == ["credit", "reserve", "reserve", "commit", "release"] + if workspace: + totals = authority.call("usage", context=governance.context_dict(tenant), role="admin") + assert totals["requests"] == 2 and totals["holds_micro_usd"] == 0 and totals["spent_micro_usd"] == 30 + budget.BudgetService.reset() + + +def test_background_item_cannot_fall_back_to_legacy_workspace(credits, monkeypatch): + app, store, admission, _ = credits + accounting = importlib.import_module("services.request_accounting") + monkeypatch.setenv("ORGANISATIONS_ENABLED", "true") + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + with patch.object(store, "read", side_effect=AssertionError("unverified item touched credits")): + error, identity = accounting.admit_item(g.authenticated_user, "mimo:mimo-v2.5", 1) + assert error["status"] == 503 and error["code"] == "tenant_storage_unavailable" + assert identity is None + + +def test_revision_mismatch_stops_after_three_total_attempts(credits): + app, store, admission, _ = credits + ledger = importlib.import_module("services.credits_ledger") + fund(store) + with app.test_request_context(): + g.authenticated_user = {"username": "alice"} + with patch.object(store, "read", wraps=store.read) as reads, patch.object( + store, "append", side_effect=ledger.CreditsError("credits_revision_mismatch", 412)) as writes: + with pytest.raises(admission.CreditAdmissionError) as error: + admission.reserve(0.0001, scoped_id="attempt") + assert error.value.code == "credits_unavailable" + assert reads.call_count == writes.call_count == 3 + assert len({call.args[3] for call in writes.call_args_list}) == 1 + assert len({call.kwargs["scoped_id"] for call in writes.call_args_list}) == 1 + assert store.read("alice")["held_microusd"] == 0 + + +@pytest.mark.parametrize("workspace", [False, True]) +def test_gateway_subrequests_replace_outer_holds_once(credits, tmp_path, monkeypatch, workspace): + from flask import Response + app, ledger, admission, contracts = credits + accounting = importlib.import_module("services.request_accounting") + dispatch = importlib.import_module("services.accounted_dispatch") + governance = importlib.import_module("services.tenant_governance") + budget = importlib.import_module("services.budget_service") + tenant = contracts.TenantContext("alice", "org", "team") if workspace else contracts.TenantContext("alice") + authority = None + if workspace: + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + authority = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path)) + governance.register_governance_collaborators(app=app, store=authority, tenant_resolver=lambda user: tenant) + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + fund(ledger, admission.owner(tenant), 10000) + budget.BudgetService.reset() + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice"} + g.tenant_context = tenant + assert accounting.begin() is None + outer = g.usage_context + dispatch.release_outer_accounting() + dispatch.release_outer_accounting() + def send(payload): + accounting.mark_dispatched() + return Response(json.dumps({"usage": {"prompt_tokens": 10, "completion_tokens": 20}}), mimetype="application/json") + for _ in range(2): + assert dispatch.accounted_dispatch(body(), send, kind="chat", skip_rate=True).status_code == 200 + assert outer.finished + entries = ledger.read(admission.owner(tenant)) + assert entries["held_microusd"] == 0 and entries["balance_microusd"] == 9940 + reserves = [e for e in entries["entries"] if e["kind"] == "reserve"] + assert len(reserves) == 3 and len({e["scoped_id"] for e in reserves}) == 3 + assert len([e for e in entries["entries"] if e["kind"] == "release"]) == 1 + if authority: + totals = authority.call("usage", context=governance.context_dict(tenant), role="admin") + assert totals["spent_micro_usd"] == 60 and totals["holds_micro_usd"] == 0 + budget.BudgetService.reset() + + +def test_local_exact_cache_namespaces_keep_legacy_bytes(monkeypatch): + cache = importlib.import_module("routes.chat_cache") + contracts = importlib.import_module("services.enterprise_contract") + app = Flask(__name__) + payload = body() + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + expected = hashlib.sha256("\0".join(("alice", "/v1/chat/completions", canonical)).encode()).hexdigest() + with app.test_request_context(): + assert cache.cache_key("alice", "/v1/chat/completions", payload) == expected + keys = [] + for org in ("one", "two"): + g.tenant_context = contracts.TenantContext("alice", org) + keys.append(cache.cache_key("alice", "/v1/chat/completions", payload)) + assert len(set([expected, *keys])) == 3 + + +def test_signed_background_capability_captures_verified_workspace(monkeypatch): + signing = importlib.import_module("services.media_signing") + hierarchy = importlib.import_module("services.tenant_hierarchy") + contracts = importlib.import_module("services.enterprise_contract") + monkeypatch.setenv("MEDIA_SIGNING_SECRET", "synthetic-background-secret") + monkeypatch.setenv("ORGANISATIONS_ENABLED", "true") + app = Flask(__name__) + tenant = contracts.TenantContext("alice", "submitted-org", "team", 7) + with app.test_request_context(): + g.tenant_context = tenant + token = signing.issue_principal("batch", "job", "alice", 60) + with app.test_request_context(): + claims = signing.read_principal(token, "batch") + g.authenticated_user = {"username": "alice"} + with patch.object(hierarchy, "resolve_principal", side_effect=AssertionError("Background resolved workspace")): + signing.bind_principal_tenant(claims, g.authenticated_user) + assert hierarchy.tenant_context_hook() is None + assert hierarchy.current_tenant() == tenant + + +def test_background_capability_without_workspace_fails_closed(monkeypatch): + signing = importlib.import_module("services.media_signing") + monkeypatch.setenv("MEDIA_SIGNING_SECRET", "synthetic-background-secret") + monkeypatch.setenv("ORGANISATIONS_ENABLED", "false") + token = signing.issue_principal("batch", "job", "alice", 60) + claims = signing.read_principal(token, "batch") + assert set(claims) == {"k", "s", "o", "e"} + monkeypatch.setenv("ORGANISATIONS_ENABLED", "true") + with Flask(__name__).test_request_context(): + with pytest.raises(Exception) as error: + signing.bind_principal_tenant(claims, {"username": "alice"}) + assert error.value.status_code == 503 + + +def test_governance_reconciliation_repairs_credit_settlement_after_restart(credits, tmp_path, monkeypatch): + app, ledger, admission, contracts = credits + governance = importlib.import_module("services.tenant_governance") + reservations = importlib.import_module("services.reservation_store") + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + authority = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path)) + context = contracts.TenantContext("alice", "submitted", "team", 4) + governance.register_governance_collaborators(app=app, store=authority, + tenant_resolver=lambda _: pytest.fail("reconciliation resolved a tenant")) + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + fund(ledger, admission.owner(context), 1000) + identity = "tg_restart" + authority.call("reserve", id=identity, context=governance.context_dict(context), amount=100, + day="2026-10-09", month="2026-10", key_daily=None, key_monthly=None, base_day=0, base_month=0) + authority.call("dispatch", id=identity) + authority.call("settle", id=identity, cost=None) + with app.test_request_context(): + hold = admission.reserve(0.0001, scoped_id=identity, context=context) + admission.mark_dispatched(hold) + admission.complete(hold, {}) + with app.app_context(): + reservations.reconcile(None, identity, 2, None, admin=True, reason="review", + evidence="provider-check", transition_id="a" * 32) + assert ledger.read(admission.owner(context))["held_microusd"] == 100 + values = dict(admin=True, reason="review", evidence="provider-check", transition_id="b" * 32) + with patch.object(admission, "reconcile_reservation", side_effect=admission.CreditAdmissionError()): + with pytest.raises(admission.CreditAdmissionError): + reservations.reconcile(None, identity, 3, 0.000025, **values) + for _ in range(2): + result = reservations.reconcile(None, identity, 3, 0.000025, **values) + assert result["applied"] is False + balance = ledger.read(admission.owner(context)) + assert balance["balance_microusd"] == 975 and balance["held_microusd"] == 0 + assert len([entry for entry in balance["entries"] if entry["kind"] == "commit"]) == 1 + totals = authority.call("usage", context=governance.context_dict(context), role="admin") + assert totals["spent_micro_usd"] == 25 and totals["holds_micro_usd"] == 0 + + +def test_subrequest_flags_off_preserve_legacy_budget_storage_calls(monkeypatch): + from flask import Response + from services import accounted_dispatch as dispatch, request_accounting as accounting + from services.budget_service import BudgetDecision, BudgetService + monkeypatch.setenv("USAGE_RESERVATIONS_ENABLED", "true") + app = Flask(__name__) + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice", "daily_budget_usd": 1} + with patch.object(accounting, "estimate_cost", return_value=0.002) as estimate, \ + patch.object(accounting, "reservation_price", side_effect=AssertionError("legacy pricing changed")), \ + patch.object(BudgetService, "check_and_reserve", return_value=BudgetDecision(True, reservation="legacy")) as reserve, \ + patch.object(accounting, "_record") as record, \ + patch.object(accounting.credits_admission, "reserve", side_effect=AssertionError("disabled credits storage")), \ + patch.object(accounting.tenant_governance, "store", side_effect=AssertionError("disabled governance storage")): + response = dispatch.accounted_dispatch(body(), lambda _: Response("{}", mimetype="application/json"), kind="chat", skip_rate=True) + assert response.status_code == 200 + estimate.assert_called_once() + reserve.assert_called_once_with(g.authenticated_user, 0.002) + assert record.call_args.args[0].reservation == "legacy" + + +def test_subrequest_predispatch_refusal_releases_credits_and_governance(credits, tmp_path, monkeypatch): + from error_handlers import APIError + from services import accounted_dispatch as dispatch, tenant_governance as governance + app, ledger, admission, contracts = credits + context = contracts.TenantContext("alice", "org") + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + authority = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path)) + governance.register_governance_collaborators(app=app, store=authority, tenant_resolver=lambda _: context) + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + fund(ledger, admission.owner(context), 1000) + def refuse(_): + raise APIError("before dispatch", 503) + with app.test_request_context("/v1/chat/completions", method="POST", json=body()): + g.authenticated_user = {"username": "alice"} + g.tenant_context = context + with pytest.raises(APIError): + dispatch.accounted_dispatch(body(), refuse, kind="chat", skip_rate=True) + balance = ledger.read(admission.owner(context)) + assert balance["held_microusd"] == 0 and balance["balance_microusd"] == 1000 + assert authority.call("usage", context=governance.context_dict(context), role="admin")["holds_micro_usd"] == 0 diff --git a/tests/test_wave7_registrar.py b/tests/test_wave7_registrar.py new file mode 100644 index 00000000..42d2dfec --- /dev/null +++ b/tests/test_wave7_registrar.py @@ -0,0 +1,490 @@ +"""Registered enterprise collaborators and processor credit boundaries.""" +import importlib +import hashlib +import hmac +import json +import os +import sqlite3 +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from tests.unified_api_test_case import UnifiedApiTestCase + +ROOT = Path(__file__).resolve().parents[1] +FLAGS = ('ORGANISATIONS_ENABLED', 'TENANT_GOVERNANCE_ENABLED', 'SAML_ENABLED', 'SCIM_ENABLED', + 'CREDITS_ENABLED', 'CREDITS_ENFORCEMENT', 'PAYMENTS_ENABLED') + + +def test_enforcement_parser(monkeypatch, caplog): + ledger = importlib.import_module('services.credits_ledger') + ledger._warned.discard('CREDITS_ENFORCEMENT') + for raw in ('', 'off', 'funded', 'all', ' ALL '): + assert ledger.enforcement({'CREDITS_ENABLED': 'true', 'CREDITS_ENFORCEMENT': raw}) == (raw.strip().lower() or 'off') + for _ in range(2): + assert ledger.enforcement({'CREDITS_ENABLED': 'true', 'CREDITS_ENFORCEMENT': 'private-invalid'}) == 'off' + assert caplog.text.count('Invalid CREDITS_ENFORCEMENT') == 1 + assert 'private-invalid' not in caplog.text + assert ledger.enforcement({'CREDITS_ENFORCEMENT': 'all'}) == 'off' + + +def test_processor_refund_and_replay(tmp_path, monkeypatch): + ledger = importlib.import_module('services.credits_ledger') + payments = importlib.import_module('services.credits_payments') + contract = importlib.import_module('services.enterprise_contract') + monkeypatch.setenv('CREDITS_ENABLED', 'true') + monkeypatch.setenv('CREDITS_CURRENCY', 'USD') + store = ledger.SqlCreditsLedger(tmp_path / 'credits.sqlite3', initialize=True) + context = contract.TenantContext('alice', 'org', 'team') + owner = ledger.context_owner(context) + callback = payments.payment_callback(store) + def event(kind, op, amount, revision): + return contract.PaymentEvent(context, 'checkout', revision, op, 'stripe', op, kind, 'USD', amount, True) + credit = event('credit', 'credit:checkout', 100, 8) + assert callback(credit).revision == 9 + store.append(owner, 'reserve', 100, 'hold', 1, scoped_id='attempt', tariff_revision=1) + store.append(owner, 'commit', 100, 'spend', 2, scoped_id='attempt', tariff_revision=1) + refund = event('refund', 'refund:checkout:100', 100, 9) + assert callback(refund).revision == 10 + assert callback(refund).revision == 10 + summary = store.read(owner) + assert (summary['balance_microusd'], summary['held_microusd'], summary['revision']) == (-100, 0, 4) + with pytest.raises(ledger.CreditsError, match='invalid_credits_request'): + store.append(owner, 'adjust', -1, 'admin', 4, actor='processor:stripe', reason='refund') + with pytest.raises(ledger.CreditsError, match='credits_insufficient'): + store.append(owner, 'reserve', 1, 'denied', 4, scoped_id='next', tariff_revision=1) + + +def test_callback_retries_only_revision_mismatch(monkeypatch): + ledger = importlib.import_module('services.credits_ledger') + payments = importlib.import_module('services.credits_payments') + contract = importlib.import_module('services.enterprise_contract') + monkeypatch.setenv('CREDITS_ENABLED', 'true') + monkeypatch.setenv('CREDITS_CURRENCY', 'USD') + store = Mock(read=Mock(side_effect=[{'revision': 5}, {'revision': 6}]), + append_payment=Mock(side_effect=[ledger.CreditsError('credits_revision_mismatch', 412), {'revision': 7}])) + event = contract.PaymentEvent(contract.TenantContext('alice'), 'checkout', 42, 'refund:checkout', 'stripe', 'event', 'refund', 'USD', 20, True) + assert payments.payment_callback(store)(event).revision == 43 + assert [call.args[4] for call in store.append_payment.call_args_list] == [5, 6] + assert len({call.args[3] for call in store.append_payment.call_args_list}) == 1 + store.read.side_effect = None + store.read.return_value = {'revision': 6} + for code in ('credits_conflict', 'credits_revision_mismatch'): + store.reset_mock() + store.append_payment.side_effect = ledger.CreditsError(code, 409) + with pytest.raises(importlib.import_module('services.credits_admission').CreditAdmissionError, match='credits_unavailable'): + payments.payment_callback(store)(event) + assert store.append_payment.call_count == store.read.call_count == 1 + + +def test_private_endpoint_bounds_and_version(monkeypatch): + transport = importlib.import_module('services.intelligence_d1_store') + state = importlib.import_module('services.control_state_d1') + bounds = {'tenant_governance': (32768, '/v1/tenant-governance'), 'saml': (8192, '/v1/managed-state/saml'), + 'scim': (131072, '/v1/managed-state/scim'), 'credits': (16384, '/v1/managed-state/credits'), + 'payments': (65536, '/v1/managed-state/payments')} + for name, (bound, path) in bounds.items(): + assert transport._ENDPOINTS[name] == 'http://intelligence.internal' + path + assert transport._ENDPOINT_MAX_BYTES[name] == bound + assert transport._RESPONSE_MAX_BYTES[transport._ENDPOINTS[name]] == bound + call = Mock(return_value={}) + monkeypatch.setattr(transport, 'request_private_intelligence', call) + state.call('credits', 'read', owner='alice') + assert call.call_args.args[0] == {'version': 1, 'operation': 'read', 'owner': 'alice'} + + +@pytest.mark.parametrize('endpoint', ['tenant_governance', 'saml', 'scim', 'credits', 'payments']) +def test_private_transport_pins_origin_and_enforces_each_bound(monkeypatch, endpoint): + private = importlib.import_module('services.intelligence_d1_store') + calls = [] + def submit(target, body, stopped, deadline, results, slots, statuses, timeout): + calls.append((target, len(body))) + results.put_nowait((True, {'version': 1})) + slots.release() + monkeypatch.setattr(private, '_submit', submit) + bound = private._ENDPOINT_MAX_BYTES[endpoint] + overhead = len(json.dumps({'data': '', 'version': 1}, separators=(',', ':')).encode()) + assert private.request_private_intelligence({'data': 'x' * (bound - overhead)}, endpoint=endpoint) == {'version': 1} + assert calls == [(private._ENDPOINTS[endpoint], bound)] + for destination, size in ((endpoint, bound - overhead + 1), (private._ENDPOINTS[endpoint], 0), + ('https://external.invalid', 0)): + with pytest.raises(private.GatewayError): + private.request_private_intelligence({'data': 'x' * size}, endpoint=destination) + assert len(calls) == 1 + + +class RegisteredEnterpriseTest(UnifiedApiTestCase): + def setUp(self): + settings = patch.dict(os.environ, {**dict.fromkeys(FLAGS, ''), + 'CONFIG_REVISION_SYNC_ENABLED': 'false', 'USAGE_LEDGER_ENABLED': 'false', + 'INTELLIGENCE_STORAGE_BACKEND': '', 'AUTH_STORAGE_BACKEND': '', 'CF_ACCESS_SSO_ONLY': 'false'}) + settings.start() + self.addCleanup(settings.stop) + for target in ('env_loader.load_runtime_env', 'config.load_runtime_env', 'services.usage_ledger.start'): + loader = patch(target) + loader.start() + self.addCleanup(loader.stop) + network = patch('requests.sessions.Session.send', side_effect=AssertionError('No network')) + network.start() + self.addCleanup(network.stop) + super().setUp() + os.environ['USAGE_DB_PATH'] = str(Path(self.temp_dir.name) / 'usage.sqlite3') + self.auth = self.app_module.AuthService + + def rebuild(self): + with patch.object(self.app_module, 'load_runtime_env'): + self.app = self.app_module.create_app() + self.app.config.update(TESTING=True, WTF_CSRF_ENABLED=False, IMAGE_RELAY_CATALOG_AUTO_REFRESH=False) + self.client = self.app.test_client() + + def login(self): + result = self.client.post('/login', data={'username': 'admin', 'api_key': 'admin-test-key'}) + assert result.status_code == 302 + + def migrate(self, path, *names): + with sqlite3.connect(path) as db: + for name in names: + db.executescript((ROOT / 'intelligence-migrations' / name).read_text()) + + def workspace(self, role='admin'): + self.migrate(os.environ['AUTH_DB_PATH'], '0033_tenant_hierarchy.sql') + tenants = importlib.import_module('services.tenant_hierarchy') + existing = self.auth._load_user_by_username('admin') + assert existing is not None + # The existing SQLite membership writer calls the schema-initializing account + # lookup under its write lock. Supply the independently verified account fixture. + store = tenants.TenantStore(principal_exists=lambda principal: principal == existing['username']) + org = store.request('org_create', data={'name': 'Workspace'}, actor='admin') + store.request('member_set', org_id=org['id'], principal='admin', actor='admin', revision=0, + data={'role': role, 'bind': True, 'binding_revision': 0}) + return store, org['id'] + + def test_all_off_preserves_hooks_sessions_generation_and_never_calls_enterprise_storage(self): + adapters = self.app.extensions['enterprise_adapters'] + assert adapters.credit is None and adapters.identity is None and adapters.payment is None + assert [hook.__name__ for hook in self.app.extensions['gateway_after_authentication']] == \ + importlib.import_module('tests.test_wave6_registrar').ORDER + assert 'tenant_context_hook' not in [hook.__name__ for hook in self.app.extensions['gateway_after_authentication']] + urls = ['/auth/saml/login', '/auth/saml/callback', '/auth/saml/acs', '/auth/saml/metadata', + '/admin/saml/links', '/scim/v2/Users', '/v1/credits', '/admin/credits/admin', + '/admin/credits/adjust', '/v1/payments/checkout', '/v1/payments/webhook', + '/admin/organisations/org/governance', '/v1/workspaces'] + transport = importlib.import_module('services.intelligence_d1_store') + tenants = importlib.import_module('services.tenant_hierarchy') + with patch.object(transport, 'request_private_intelligence', side_effect=AssertionError('Disabled storage')), \ + patch.object(tenants.TenantStore, 'request', side_effect=AssertionError('Disabled tenants')): + for url in urls: + assert self.client.get(url).status_code == 404, url + with patch.object(self.app_module.ProxyService, 'make_request', return_value=self._chat_response()) as upstream: + result = self.client.post('/v1/chat/completions', headers={'Authorization': 'Bearer admin-test-key'}, + json={'model': 'opencode:test', 'messages': []}) + assert result.status_code == 200 and upstream.call_count == 1 + self.login() + with self.client.session_transaction() as saved: + assert set(saved) == {'authenticated', 'user'} + assert set(saved['user']) == {'username', 'is_admin', 'api_key_prefix', 'scopes', 'session_id'} + assert self.client.get('/users').status_code == 200 + + def test_workspace_list_and_switch_use_registered_context(self): + os.environ['ORGANISATIONS_ENABLED'] = 'true' + self.rebuild() + store, org = self.workspace() + headers = {'Authorization': 'Bearer admin-test-key'} + result = self.client.get('/v1/workspaces', headers=headers) + assert result.status_code == 200 and result.json['binding']['org_id'] == org + switched = self.client.post('/v1/workspaces/switch', headers={**headers, 'If-Match': '"1"'}, + json={'org_id': org, 'team_id': None}) + assert switched.status_code == 200 and switched.json['revision'] == 2 + store.request('org_update', org_id=org, actor='admin', revision=1, data={'status': 'deactivated'}) + assert self.client.get('/v1/credits', headers=headers).status_code == 404 + assert self.client.get('/v1/models', headers=headers).status_code == 403 + + def test_governance_role_and_private_cas_through_real_routes(self): + os.environ.update(ORGANISATIONS_ENABLED='true', TENANT_GOVERNANCE_ENABLED='true') + self.rebuild() + store, org = self.workspace() + self.login() + transport = importlib.import_module('services.intelligence_d1_store') + state = importlib.import_module('services.control_state_d1') + revision = 0 + def private(document, *, endpoint): + nonlocal revision + assert endpoint == 'tenant_governance' and document['version'] == 1 and document['rpc'] is True + if document['operation'] == 'put': + if document['revision'] != revision: + raise transport.PrivateIntelligenceError(412, 'revision_stale') + revision += 1 + return {'version': 1, 'policy': {'revision': revision, 'models': None, 'tools': None, 'daily': 100, 'monthly': None}} + path = '/admin/organisations/' + org + '/governance' + with patch.object(state, 'using_d1', return_value=True), patch.object(transport, 'request_private_intelligence', side_effect=private): + assert self.client.get(path).headers['ETag'] == '"0"' + assert self.client.put(path, json={'daily': 100}).status_code == 428 + result = self.client.put(path, headers={'If-Match': '"0"'}, json={'daily': 100}) + assert result.status_code == 200 and result.headers['ETag'] == '"1"' + assert self.client.put(path, headers={'If-Match': '"0"'}, json={'daily': 100}).status_code == 412 + store.request('member_set', org_id=org, principal='admin', actor='admin', revision=1, data={'status': 'deactivated'}) + assert self.client.get(path).status_code == 403 + + def test_registered_governance_tools_intersect_key_and_ancestor_grants(self): + from flask import g + governance = importlib.import_module('services.tenant_governance') + deferred = importlib.import_module('services.deferred_tools') + contract = importlib.import_module('services.enterprise_contract') + os.environ.update(TENANT_GOVERNANCE_ENABLED='true', ORGANISATIONS_ENABLED='true') + rows = [{'principal_id': 'alice', 'tool_name': name, 'allowed': allowed, 'revision': 1, + 'scopes': ['models']} for name, allowed in [('allowed', 1), ('ancestor_denied', 1), ('key_denied', 0)]] + with patch.object(deferred.D1GrantReader, '__call__', return_value=rows): + self.rebuild() + collaborators = self.app.extensions['tenant_governance'] + store = Mock(call=Mock(return_value={'policies': [{'tools': ['allowed', 'key_denied']}, {'tools': ['*']}]})) + governance.register_governance_collaborators(app=self.app, store=store, + tenant_resolver=collaborators.tenant_resolver, membership_role=collaborators.membership_role) + entries = [{'scope': 'models', 'definition': {'name': row['tool_name'], 'inputSchema': {'type': 'object'}}} for row in rows] + with self.app.test_request_context('/mcp'): + g.authenticated_user = {'username': 'alice', 'scopes': ['models']} + g.tenant_context = contract.TenantContext('alice', 'org') + service = self.app.extensions['deferred_tools'] + assert [item['name'] for item in service.list_tools(entries, g.authenticated_user)] == ['allowed'] + with pytest.raises(Exception, match='not granted'): + service.require_grant(entries[1], g.authenticated_user) + + def test_saml_registered_issuer_matches_password_session_and_denies_inactive_or_unlinked(self): + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import ed25519 + helpers = importlib.import_module('tests.test_saml_federation') + key = ed25519.Ed25519PrivateKey.generate() + signer = ('EdDSA', key, key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo).decode()) + os.environ.update(helpers.settings(signer)) + self.rebuild() + self.login() + with self.client.session_transaction() as saved: + expected = dict(saved) + expected['user'] = dict(saved['user']) + self.client = self.app.test_client() + os.environ['INTELLIGENCE_STORAGE_BACKEND'] = 'd1' + service = self.app.extensions['saml_federation'] + service.clock = lambda: helpers.NOW + store = helpers.Store() + transport = importlib.import_module('services.intelligence_d1_store') + def private(document, *, endpoint): + assert endpoint == 'saml' + return store(document) + with patch.object(transport, 'request_private_intelligence', side_effect=private): + service.put_link({'issuer': helpers.ISSUER, 'subject': 'external:alice', 'account': 'admin'}, 'admin') + claims = helpers.start(self.client) + assert helpers.callback(self.client, signer, claims).status_code == 302 + with self.client.session_transaction() as saved: + actual = dict(saved) + actual['user'] = dict(saved['user']) + assert set(actual) == set(expected) + actual['user'].pop('session_id') + expected['user'].pop('session_id') + assert actual == expected + assert helpers.callback(self.client, signer, claims).status_code == 400 + self.client = self.app.test_client() + claims = helpers.start(self.client) + claims['sub'] = 'unlinked' + assert helpers.callback(self.client, signer, claims).status_code == 403 + with self.client.session_transaction() as saved: + assert 'authenticated' not in saved + claims = helpers.start(self.client) + record = self.auth.get_user_record('admin') + with patch.object(self.auth, 'get_user_record', return_value={**record, 'revoked_at': 'revoked'}): + assert helpers.callback(self.client, signer, claims).status_code == 403 + + def test_credits_read_and_admin_adjust_share_the_registered_ledger(self): + os.environ['CREDITS_ENABLED'] = 'true' + self.migrate(os.environ['USAGE_DB_PATH'], '0037_credits_ledger.sql') + self.rebuild() + self.login() + result = self.client.post('/admin/credits/adjust', headers={'If-Match': '"0"'}, + json={'owner': 'admin', 'amount_microusd': 100, 'operation_id': 'allocation', 'reason': 'Reviewed'}) + assert result.status_code == 200 + result = self.client.get('/v1/credits', headers={'Authorization': 'Bearer admin-test-key'}) + assert result.status_code == 200 and result.json['balance_microusd'] == 100 + assert self.client.get('/admin/credits/admin').headers['ETag'] == '"1"' + + def test_scim_registered_d1_create_patch_deactivate_and_backend_gate(self): + helpers = importlib.import_module('tests.test_scim_provisioning') + scim = importlib.import_module('services.scim_provisioning') + transport = importlib.import_module('services.intelligence_d1_store') + os.environ.update(SCIM_ENABLED='true', SCIM_TEST_A='synthetic-scim-a', + SCIM_TRUST_CONFIG_JSON=json.dumps({'tenants': {'org:a': {'token_ref': 'SCIM_TEST_A'}}})) + store = helpers.Store() + calls = [] + def private(document, *, endpoint): + calls.append(document) + assert endpoint == 'scim' and document['version'] == 1 + args = dict(document) + operation = args.pop('operation') + args.pop('version') + if operation == 'probe': + store.probe() + result = {} + elif operation == 'account': + result = {'account': store.account(args['username'])} + elif operation == 'list': + rows, total = store.list(args['org_id'], args['kind'], args['attribute'], args['value'], args['start'], args['count']) + result = {'resources': rows, 'total': total} + elif operation == 'get': + result = {'resource': store.get(args['org_id'], args['kind'], args['id'])} + else: + resource, created = store.put(args.pop('org_id'), args.pop('kind'), **args) + result = {'resource': resource, 'created': created} + return {'version': 1, **result} + with patch.object(transport, 'request_private_intelligence', side_effect=private): + self.rebuild() + assert helpers.user(self.client).status_code == 503 + assert calls == [] + os.environ.update(AUTH_STORAGE_BACKEND='d1', INTELLIGENCE_STORAGE_BACKEND='d1', + CONFIG_REVISION_SYNC_ENABLED='true') + with patch.object(self.auth, '_load_user_by_username', side_effect=store.account): + created = helpers.user(self.client) + assert created.status_code == 201 + path = 'Users/' + created.json['id'] + changed = helpers.call(self.client, 'PATCH', path, {'schemas': [scim.PATCH_SCHEMA], + 'Operations': [{'op': 'replace', 'path': 'displayName', 'value': 'Alice'}]}, + **{'If-Match': created.headers['ETag']}) + assert changed.status_code == 200 and changed.json['displayName'] == 'Alice' + os.environ['ORGANISATIONS_ENABLED'] = 'true' + tenant_store = Mock(request=Mock(return_value={'id': 'org:a', 'status': 'active'})) + self.app.extensions['tenant_store'] = tenant_store + group = {'schemas': [scim.GROUP_SCHEMA], 'displayName': 'Team', + 'members': [{'value': created.json['id']}]} + assert helpers.call(self.client, 'POST', 'Groups', group).status_code == 201 + tenant_store.request.assert_called_with('org_get', org_id='org:a') + assert helpers.call(self.client, 'DELETE', path).status_code == 204 + assert store.accounts['alice']['revoked_at'] + assert helpers.call(self.client, 'GET', path).json['active'] is False + group['displayName'] = 'Other team' + assert helpers.call(self.client, 'POST', 'Groups', group).status_code == 400 + assert len(store.accounts) == 1 + assert all('api_key' not in str(row.get('resource', {})) for row in calls) + + def test_payment_checkout_alone_is_mounted_and_credit_failure_stays_pending(self): + self.payment_setup(workspace=False) + checkout = self.checkout() + raw = self.paid(checkout) + result = self.deliver(raw) + assert result.status_code == 503 and result.json['error']['code'] == 'payment_credit_unavailable' + with sqlite3.connect(os.environ['USAGE_DB_PATH']) as db: + assert db.execute('SELECT status FROM payment_events').fetchone()[0] == 'pending_credit' + + def payment_setup(self, *, workspace=True): + helpers = importlib.import_module('tests.test_payment_billing') + os.environ.update(helpers.ENV) + if workspace: + os.environ.update(ORGANISATIONS_ENABLED='true', CREDITS_ENABLED='true', TENANT_GOVERNANCE_ENABLED='true') + self.migrate(os.environ['USAGE_DB_PATH'], '0037_credits_ledger.sql', '0038_payment_billing.sql') + self.rebuild() + if workspace: + _, self.org = self.workspace(role='billing') + service = self.app.extensions['payment_billing'] + service.secret_resolver = lambda ref: helpers.SECRET + service.clock = lambda: helpers.NOW + def http(url, *, headers, body): + from urllib.parse import parse_qs + fields = parse_qs(body) + return {'id': 'cs_' + fields['client_reference_id'][0], 'object': 'checkout.session', + 'url': 'https://checkout.stripe.com/c/test', 'amount_total': 50, 'currency': 'usd', 'livemode': False} + service.http = http + + def checkout(self): + body = importlib.import_module('tests.test_payment_billing').BODY + result = self.client.post('/v1/payments/checkout', headers={'Authorization': 'Bearer admin-test-key'}, json=body) + assert result.status_code == 200, result.json + return result.json + + def paid(self, checkout): + return json.dumps({'id': 'evt_paid', 'object': 'event', 'type': 'checkout.session.completed', 'livemode': False, + 'data': {'object': {'id': 'cs_' + checkout['checkout_id'], 'client_reference_id': checkout['checkout_id'], + 'metadata': {'checkout_id': checkout['checkout_id']}, 'payment_status': 'paid', 'amount_total': 50, + 'currency': 'usd', 'livemode': False, 'payment_intent': 'pi_test'}}}, indent=2).encode() + b'\n' + + def deliver(self, raw): + helpers = importlib.import_module('tests.test_payment_billing') + signature = hmac.new(helpers.SECRET.encode(), str(helpers.NOW).encode() + b'.' + raw, hashlib.sha256).hexdigest() + return self.client.post('/v1/payments/webhook', data=raw, content_type='application/json', + headers={'Stripe-Signature': f't={helpers.NOW},v1={signature}'}) + + def test_workspace_checkout_signed_credit_refund_and_replay_have_one_owner_and_exact_bytes(self): + self.payment_setup() + checkout = self.checkout() + helpers = importlib.import_module('tests.test_payment_billing') + payments = importlib.import_module('services.payment_billing') + ledger = importlib.import_module('services.credits_ledger') + contract = importlib.import_module('services.enterprise_contract') + owner = ledger.context_owner(contract.TenantContext('admin', self.org)) + raw = self.paid(checkout) + original = payments.verify_signature + with patch.object(payments, 'verify_signature', wraps=original) as verify: + assert self.deliver(raw).status_code == 200 + assert verify.call_args.args[0] == raw + assert self.deliver(raw).status_code == 200 + result = self.client.get('/v1/credits', headers={'Authorization': 'Bearer admin-test-key'}) + assert result.status_code == 200 and result.json['balance_microusd'] == 500000 + assert ledger.open_store().read('admin')['balance_microusd'] == 0 + store = ledger.open_store() + store.append(owner, 'reserve', 500000, 'hold', 1, scoped_id='attempt', tariff_revision=1) + store.append(owner, 'commit', 500000, 'spent', 2, scoped_id='attempt', tariff_revision=1) + refund = json.dumps({'id': 'evt_refund', 'object': 'event', 'type': 'charge.refunded', 'livemode': False, + 'data': {'object': {'id': 'ch_test', 'payment_intent': 'pi_test', 'amount': 50, 'amount_refunded': 50, + 'currency': 'usd', 'livemode': False}}}).encode() + assert self.deliver(refund).status_code == 200 + assert self.deliver(refund).status_code == 200 + assert store.read(owner)['balance_microusd'] == -500000 + assert store.read(owner)['revision'] == 4 + self.login() + with patch.object(self.auth, 'get_current_user', return_value={'username': 'processor:stripe', 'is_admin': True}): + denied = self.client.post('/admin/credits/adjust', headers={'If-Match': '"4"'}, + json={'owner': owner, 'amount_microusd': -1, 'operation_id': 'spoof', 'reason': 'refund'}) + assert denied.status_code == 400 + + def test_all_flags_signed_webhook_keeps_cached_bytes_and_bypasses_key_freshness(self): + from flask import request + self.payment_setup() + checkout = self.checkout() + raw = self.paid(checkout) + flags = importlib.import_module('tests.test_wave5_registrar').FLAGS + values = {name: 'block' if name == 'PROMPT_INJECTION_MODE' else + 'shadow' if name == 'BANDIT_MODE' else 'true' for name in flags} + os.environ.update(values) + os.environ.update(importlib.import_module('tests.test_wave6_registrar').FLAGS) + os.environ.update({name: 'all' if name == 'CREDITS_ENFORCEMENT' else 'true' for name in FLAGS}) + registrar = importlib.import_module('services.gateway_extensions') + sync = Mock(settings=Mock(enabled=True), security_ready=Mock(return_value=False)) + with patch.object(registrar, 'supported_settings', side_effect=lambda settings: settings), \ + patch.object(registrar, 'configure_sync', return_value=sync): + self.rebuild() + helpers = importlib.import_module('tests.test_payment_billing') + service = self.app.extensions['payment_billing'] + service.secret_resolver = lambda ref: helpers.SECRET + service.clock = lambda: helpers.NOW + def cache_json(): + if request.endpoint == 'payment_webhook': + assert request.get_json()['id'] == 'evt_paid' + self.app.before_request_funcs[None].insert(0, cache_json) + payments = importlib.import_module('services.payment_billing') + with patch.object(payments, 'verify_signature', wraps=payments.verify_signature) as verify: + result = self.deliver(raw) + assert result.status_code == 200, result.json + assert verify.call_args.args[0] == raw + sync.security_ready.assert_not_called() + + +@pytest.mark.parametrize("composed_first", [False, True]) +def test_preview_preserves_composed_enterprise_adapters(composed_first): + from flask import Flask + routes = importlib.import_module("routes.enterprise_preview") + contracts = importlib.import_module("services.enterprise_contract") + app = Flask(__name__) + composed = contracts.register_enterprise_adapters(tenant=lambda operation: operation.context) + def compose(): + app.extensions["enterprise_adapters"] = composed + if composed_first: + compose() + routes.register_enterprise_preview_routes(app) + if not composed_first: + compose() + assert app.extensions["enterprise_adapters"] is composed diff --git a/tests/test_wave7_worker.mjs b/tests/test_wave7_worker.mjs new file mode 100644 index 00000000..7ee50c45 --- /dev/null +++ b/tests/test_wave7_worker.mjs @@ -0,0 +1,483 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { DatabaseSync } from "node:sqlite"; +import { readFileSync } from "node:fs"; +import { TenantContext, AuthorityResult } from "../worker/enterprise-contract.mjs"; +import { createCreditsLifecycle, CreditsAdmissionError } from "../worker/credits-admission.mjs"; +import { appendCredit, readCredits, contextOwner } from "../worker/credits-d1.mjs"; +import { handleTenantGovernanceRequest } from "../worker/tenant-governance-d1.mjs"; +import { handleReservationsRequest } from "../worker/reservations-d1.mjs"; +import { nativeGenerationFetch, nativeGenerationSetup } from "../worker/gateway-extensions.mjs"; +import { exactCacheIdentity } from "../worker/exact-generation-cache.mjs"; +import { semanticPartition } from "../worker/semantic-generation-cache.mjs"; +import { handleIntelligenceOutbound } from "../worker/intelligence-outbound.mjs"; +import { handleIdempotencyRequest } from "../worker/idempotency-d1.mjs"; +import { handleResponsesStateRequest } from "../worker/responses-state-d1.mjs"; +import { handleContextPageStateRequest } from "../worker/context-pages-d1.mjs"; +import { recordNativeUsage } from "../worker/usage-ledger-d1.mjs"; +import { tenantStorageKey } from "../worker/tenants-d1.mjs"; +import { retentionRequestId } from "../worker/retention-policy.mjs"; +import { USER_FIELDS } from "../worker/control-users-d1.mjs"; +import { loadWorkerModule } from "./helpers/load_cloudflare_worker.mjs"; +import { handleManagedStateRequest } from "../worker/managed-state-dispatch.mjs"; + +const request = path => new Request(`http://intelligence.internal${path}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, operation: "probe" }), +}); +test("enterprise private domains retain disabled gates and reach enabled authorities", async () => { + for (const [domain, flag] of [["saml", "SAML_ENABLED"], ["scim", "SCIM_ENABLED"], ["credits", "CREDITS_ENABLED"], ["payments", "PAYMENTS_ENABLED"]]) { + assert.equal((await handleManagedStateRequest(request(`/v1/managed-state/${domain}`), {})).status, 404); + assert.notEqual((await handleManagedStateRequest(request(`/v1/managed-state/${domain}`), { [flag]: "true", PAYMENT_PROCESSOR_CONFIG_JSON: JSON.stringify({ processor: "stripe", secret_key_ref: "PAYMENT_TEST_KEY", return_origins: ["https://portal.test"], product_name: "Gateway credits" }), PAYMENT_WEBHOOK_KEY_REF: "PAYMENT_TEST_WEBHOOK" })).status, 404); + } +}); + +const allFlags = { ORGANISATIONS_ENABLED: "true", TENANT_GOVERNANCE_ENABLED: "true", SAML_ENABLED: "true", SCIM_ENABLED: "true", + CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all", PAYMENTS_ENABLED: "true" }; +const tenant = (org = null) => new TenantContext({ principal_id: "admin", org_id: org }); +const route = "/test/v1/chat/completions"; +const payload = { model: "test", messages: [{ role: "user", content: "fixture" }], max_tokens: 8, temperature: 0 }; +const nativeRequest = (headers = {}) => new Request(`https://provider.invalid${route}`, { + method: "POST", headers: { "content-type": "application/json", ...headers }, body: JSON.stringify(payload), +}); +const completion = JSON.stringify({ choices: [{ message: { content: "fixture" }, finish_reason: "stop" }], + usage: { prompt_tokens: 3, completion_tokens: 2 } }); +const complete = () => new Response(completion, { headers: { "content-type": "application/json" } }); +const authority = { authenticated: true, provider: "openai", route, principal: { id: "admin" } }; +function fixture(t, flags = {}) { + const sql = new DatabaseSync(":memory:"); t.after(() => sql.close()); + for (const migration of ["0003_control_users.sql", "0007_usage_ledger.sql", "0011_secret_firewall.sql", "0013_shadow_eval.sql", + "0017_control_revisions.sql", "0019_generation_cache.sql", "0020_usage_reservations.sql", "0023_idempotency.sql", + "0024_context_pages.sql", "0026_semantic_cache.sql", "0028_responses_state.sql", "0033_tenant_hierarchy.sql", "0034_tenant_governance.sql", + "0035_saml_federation.sql", "0036_scim_provisioning.sql", "0037_credits_ledger.sql", "0038_payment_billing.sql"]) + sql.exec(readFileSync(new URL(`../intelligence-migrations/${migration}`, import.meta.url), "utf8")); + const operations = []; + let failBatch = false; + const db = { prepare(query) { + let values = []; + const execute = () => { + operations.push(query); + const numbered = /\?[1-9]/.test(query); + const stmt = sql.prepare(query.replace(/\?(\d+)/g, (_, n) => `$v${n}`)); + stmt.setAllowUnknownNamedParameters(true); + const args = numbered ? [Object.fromEntries(values.map((v,i) => [`v${i+1}`, v]))] : values; + const rows = stmt.columns().length ? stmt.all(...args) : (stmt.run(...args), []); + return { success: true, results: rows.map(row => ({ ...row })), meta: { changes: sql.prepare("SELECT changes() n").get().n } }; + }; + const statement = { bind(...args) { values = args; return statement; }, all: execute, run: execute, + first: () => execute().results[0] ?? null }; + return statement; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { + const results = statements.map(statement => statement.all()); + if (failBatch) { failBatch = false; throw Error("synthetic batch failure"); } + sql.exec("COMMIT"); return results; + } catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const objects = new Map(); + const bucket = { async put(key,value,options = {}) { + const bytes = typeof value === "string" ? new TextEncoder().encode(value) : new Uint8Array(value); + objects.set(key, { bytes, customMetadata: options.customMetadata }); + }, async get(key) { + const item = objects.get(key); + return item ? { size: item.bytes.length, customMetadata: item.customMetadata, arrayBuffer: async () => item.bytes.slice().buffer, + text: async () => new TextDecoder().decode(item.bytes) } : null; + }, async head(key) { return objects.has(key) ? { size: objects.get(key).bytes.length } : null; }, + async delete(key) { objects.delete(key); } }; + const env = { ADMIN_API_KEY: "synthetic-admin-key", INTELLIGENCE_DB: db, multillm_media: bucket, + MODEL_PRICING_USD_PER_MILLION: '{"openai:test":{"input":1,"output":2}}', MEDIA_SIGNING_SECRET: "synthetic-media-key", ...flags }; + const bind = (org, principal = "admin") => { + sql.prepare("INSERT OR IGNORE INTO tenant_organisations VALUES (?,?,'active',1)").run(org,org); + sql.prepare("INSERT OR IGNORE INTO tenant_memberships VALUES (?,?,NULL,'member','active',1)").run(org,principal); + sql.prepare("INSERT INTO tenant_bindings VALUES (?,?,NULL,1) ON CONFLICT(principal) DO UPDATE SET org_id=excluded.org_id,revision=revision+1") + .run(principal,org); + }; + return { sql, db, env, objects, operations, bind, failNextBatch() { failBatch = true; } }; +} +const privateRequest = (domain, body) => new Request(`http://intelligence.internal/v1/managed-state/${domain}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }), +}); +async function fund(f, context = tenant(), amount = 1_000_000) { + const owner = contextOwner(context); + await appendCredit(f.db, { owner, kind: "credit", amount_microusd: amount, operation_id: "fund", revision: 0 }); +} +for (const mode of ["funded", "all"]) test(`native ${mode} reserves and commits measured credits with metrics off`, async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: mode }); await fund(f); + let calls = 0; + const result = await nativeGenerationFetch(nativeRequest(), f.env, {}, authority, () => { calls++; return complete(); }); + assert.equal(result.status, 200); assert.equal(await result.text(), completion); assert.equal(calls,1); + const balance = await readCredits(f.db, "admin"); + assert.equal(balance.balance_microusd, 999993); assert.equal(balance.held_microusd,0); + assert.deepEqual(balance.entries.map(entry => entry.kind), ["credit","reserve","commit"]); + assert.equal(new Set(balance.entries.slice(1).map(entry => entry.scoped_id)).size,1); +}); +for (const [mode, amount] of [["off", 0], ["funded", null]]) test(`native ${mode} uncharged owner skips ledger and pricing`, async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: mode, MODEL_PRICING_USD_PER_MILLION: undefined }); + if (amount === 0) await fund(f, tenant(),0); + f.operations.length = 0; + const result = await nativeGenerationFetch(nativeRequest(), f.env, {}, authority, complete); + assert.equal(await result.text(), completion); + assert.equal(f.sql.prepare("SELECT count(*) n FROM credits_entries WHERE kind='reserve'").get().n,0); + if (mode === "off") assert.equal(f.operations.some(query => query.includes("credits_")), false); +}); +for (const [amount, priced, code, status] of [[0,true,"credits_insufficient",402], [1_000_000,false,"credits_unpriced",503]]) + test(`native ${code} denies before upstream and admission`, async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all", ADMISSION_ENABLED: "true", + ...(priced ? {} : { MODEL_PRICING_USD_PER_MILLION: undefined }), + ADMISSION_LIMITS_JSON: '{"principal":1}', ADMISSION_COORDINATOR: { getByName() { assert.fail("denial must precede admission lease"); } } }); + await fund(f, tenant(), amount); + const result = await nativeGenerationFetch(nativeRequest(), f.env, {}, authority, () => assert.fail("denied upstream")); + assert.equal(result.status,status); assert.equal((await result.json()).error.code,code); + assert.equal((await readCredits(f.db,"admin")).held_microusd,0); + }); +test("native unknown response retains only its attempt hold", async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all" }); await fund(f); + const result = await nativeGenerationFetch(nativeRequest(), f.env, {}, authority, () => new Response("{}", { headers: { "content-type": "application/json" } })); + await result.text(); + const summary = await readCredits(f.db,"admin"); + assert.ok(summary.held_microusd > 0); assert.equal(summary.balance_microusd,1_000_000); + assert.equal(JSON.parse(f.sql.prepare("SELECT document FROM credits_entries ORDER BY revision DESC LIMIT 1").get().document).unknown,true); +}); +test("credit revision retries preserve operation amount and ID and stop at three", async () => { + const seen = []; + const helper = createCreditsLifecycle({ CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all" }, { context: tenant(), + read: async () => ({ revision: seen.length }), authority: () => ({ + async reserve(operation) { + seen.push(operation); + if (seen.length < 3) throw Object.assign(Error(), { status: 412 }); + const { amount: _amount, ...result } = operation; + return new AuthorityResult({ ...result, allowed: true, revision: operation.revision+1 }); + }, commit() {}, reconcile() {}, + }) }); + await helper.admit({ amount:10, tariff_revision:7 }); + assert.equal(seen.length,3); + assert.equal(new Set(seen.map(operation => operation.operation_id)).size,1); + assert.equal(new Set(seen.map(operation => operation.scoped_id)).size,1); + assert.deepEqual(seen.map(operation => operation.amount),[10,10,10]); + assert.deepEqual(seen.map(operation => operation.revision),[0,1,2]); +}); +test("pre-dispatch cancellation releases the estimate once with a separate operation ID", async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all" }); await fund(f); + const hooks = createCreditsLifecycle(f.env, { context: tenant(), scoped_id: "attempt_one" }); + await hooks.admit({ amount:73, tariff_revision:7 }); await hooks.before_dispatch(); + await hooks.finalize({ handedOff:false }); await hooks.finalize({ handedOff:false }); + const summary = await readCredits(f.db,"admin"); + assert.equal(summary.held_microusd,0); assert.equal(summary.balance_microusd,1_000_000); + assert.deepEqual(summary.entries.slice(1).map(entry => [entry.kind, entry.amount_microusd, entry.operation_id]), + [["reserve",73,"attempt_one.reserve"],["release",73,"attempt_one.release"]]); +}); +test("all flags on reject tenant before security, policies, storage or admission", async t => { + const f = fixture(t, { ...allFlags, CONFIG_REVISION_SYNC_ENABLED:"true", ADMISSION_ENABLED:"true" }); + f.bind("org:a"); f.sql.exec("UPDATE tenant_memberships SET status='deactivated'"); + const result = await nativeGenerationFetch(nativeRequest(),f.env,{},authority,()=>assert.fail("forbidden upstream")); + assert.equal(result.status,403); assert.equal((await result.json()).error.code,"workspace_forbidden"); + assert.equal(f.operations.some(query => /credits_|governance|control_users|usage_reservations/.test(query)),false); +}); +test("organisations on keep the native generation deadline hook", () => { + const setup = nativeGenerationSetup(nativeRequest({"X-MultiLLM-Deadline-Ms":"5000"}), { ORGANISATIONS_ENABLED:"true" }); + assert.notEqual(setup.hook, null); + setup.hook.deadline?.stop(); +}); +test("all flags on governance budget denial leaves no credits, key holds, claims or admission", async t => { + const f = fixture(t, {...allFlags, ADMISSION_ENABLED:"true", USAGE_RESERVATIONS_ENABLED:"true"}); f.bind("org:a"); await fund(f,tenant("org:a")); + f.sql.prepare("INSERT INTO tenant_governance_policies (org_id,team_id,revision,models,tools,daily,monthly) VALUES (?,'',1,NULL,NULL,0,NULL)").run("org:a"); + const result = await nativeGenerationFetch(nativeRequest(),f.env,{},authority,()=>assert.fail("budget upstream")); + assert.equal(result.status,429); assert.equal((await result.json()).error.code,"tenant_budget_exceeded"); + for (const table of ["credits_entries","usage_reservations","tenant_governance_reservations","managed_idempotency"]) + assert.equal(f.sql.prepare(`SELECT count(*) n FROM ${table}${table === "credits_entries" ? " WHERE kind='reserve'" : ""}`).get().n,0); +}); +test("all flags on reserves workspace governance and credits once and settles actual micro USD", async t => { + const f = fixture(t, {...allFlags, USAGE_RESERVATIONS_ENABLED:"true"}); f.bind("org:a"); await fund(f,tenant("org:a")); + const result = await nativeGenerationFetch(nativeRequest(),f.env,{}, { ...authority, + principal: { id:"admin",daily_budget_usd:1,monthly_budget_usd:10 } }, () => { + assert.equal(f.sql.prepare("SELECT state FROM tenant_governance_reservations").get().state,"dispatched"); + assert.equal(f.sql.prepare("SELECT count(*) n FROM credits_entries WHERE kind='reserve'").get().n,1); + assert.equal(f.sql.prepare("SELECT count(*) n FROM usage_reservations").get().n,0); + return complete(); + }); + assert.equal(await result.text(),completion); + assert.equal(f.sql.prepare("SELECT charged FROM tenant_governance_reservations").get().charged,7); + assert.equal((await readCredits(f.db,contextOwner(tenant("org:a")))).held_microusd,0); +}); +test("late hook denial releases governance and credits without dispatch", async t => { + const f = fixture(t,allFlags); f.bind("org:a"); await fund(f,tenant("org:a")); + const result = await nativeGenerationFetch(nativeRequest(),f.env,{},authority,()=>assert.fail("late upstream"),[{ + enabled:()=>true, before_dispatch() { throw new CreditsAdmissionError("credits_unavailable"); }, + }]); + assert.equal(result.status,503); assert.equal((await readCredits(f.db,contextOwner(tenant("org:a")))).held_microusd,0); + assert.equal(f.sql.prepare("SELECT charged FROM tenant_governance_reservations").get().charged,0); +}); +test("governance fixed domain is reachable and grant denial precedes cache and credits", async t => { + const f = fixture(t,{ORGANISATIONS_ENABLED:"true",TENANT_GOVERNANCE_ENABLED:"true"}); f.bind("org:a"); + const res = await handleIntelligenceOutbound(new Request("http://intelligence.internal/v1/tenant-governance",{ + method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({version:1,operation:"policies",context:tenant("org:a")}), + }),f.env); assert.equal(res.status,200); + f.sql.prepare("INSERT INTO tenant_governance_policies (org_id,team_id,revision,models,tools) VALUES (?,'',1,'[]',NULL)").run("org:a"); + const denied = await nativeGenerationFetch(nativeRequest(),f.env,{},authority,()=>assert.fail("grant upstream")); + assert.equal(denied.status,403); assert.equal((await denied.json()).error.code,"model_not_allowed"); + assert.equal(f.sql.prepare("SELECT count(*) n FROM tenant_governance_reservations").get().n,0); +}); + +const scimId = n => String(n).padStart(32,"0"); +function scimUser(n, active = true, revision = 1) { + return { schemas:["urn:ietf:params:scim:schemas:core:2.0:User"], id:scimId(n),userName:`user-${n}`,active, + meta:{resourceType:"User",version:`W/"${revision}"`,created:"2026-10-09T00:00:00Z",lastModified:"2026-10-09T00:00:00Z",location:`/scim/v2/Users/${scimId(n)}`} }; +} +function scimAccount(n, changes = {}) { + return {...Object.fromEntries(USER_FIELDS.map(field=>[field,null])),username:`user-${n}`,api_key_hash:"synthetic-hash", + api_key_prefix:"mllm_synthetic",is_admin:0,scopes:"chat,models",created_at:"2026-10-09T00:00:00Z",...changes}; +} +async function scimPut(f, row, account, expected = 0, kind = "Users", deactivate = false) { + const response = await handleManagedStateRequest(privateRequest("scim", { operation:"put",org_id:"org:a",resource:row,kind, + account,expected,token_digest:null,deactivate }),f.env); + return {status:response.status,...await response.json()}; +} +test("registered SCIM creates and deactivates accounts, identities and memberships in one batch", async t => { + const f = fixture(t,{SCIM_ENABLED:"true",CONFIG_REVISION_SYNC_ENABLED:"true"}); f.bind("org:a"); + assert.equal((await scimPut(f,scimUser(1),scimAccount(1))).status,200); + assert.equal(f.sql.prepare("SELECT status FROM tenant_memberships WHERE principal='user-1'").get().status,"active"); + assert.equal(f.sql.prepare("SELECT count(*) n FROM tenant_bindings WHERE principal='user-1'").get().n,0); + const revoked = scimAccount(1,{api_key_hash:"synthetic-new-hash",api_key_prefix:"mllm_rotated",revoked_at:"2026-10-09T01:00:00Z"}); + assert.equal((await scimPut(f,scimUser(1,false,2),revoked,1)).status,200); + assert.equal(f.sql.prepare("SELECT revoked_at FROM control_users WHERE username='user-1'").get().revoked_at,revoked.revoked_at); + assert.equal(f.sql.prepare("SELECT status FROM tenant_memberships WHERE principal='user-1'").get().status,"deactivated"); + assert.equal(f.sql.prepare("SELECT revision FROM control_revisions WHERE domain='key_controls'").get().revision,2); +}); +test("SCIM batch failure rolls back account, resource, membership, audit and security revision", async t => { + const f = fixture(t,{SCIM_ENABLED:"true",CONFIG_REVISION_SYNC_ENABLED:"true"}); f.bind("org:a"); f.failNextBatch(); + assert.equal((await scimPut(f,scimUser(2),scimAccount(2))).status,503); + for (const table of ["control_users","scim_resources","scim_audit","control_revisions"]) + assert.equal(f.sql.prepare(`SELECT count(*) n FROM ${table}`).get().n,0); + assert.equal(f.sql.prepare("SELECT count(*) n FROM tenant_memberships WHERE principal='user-2'").get().n,0); +}); +test("registered SCIM groups update tenant teams atomically and reject foreign memberships", async t => { + const f = fixture(t,{SCIM_ENABLED:"true",CONFIG_REVISION_SYNC_ENABLED:"true"}); f.bind("org:a"); + await scimPut(f,scimUser(1),scimAccount(1)); + const group = { schemas:["urn:ietf:params:scim:schemas:core:2.0:Group"],id:scimId(3),displayName:"team-three",members:[{value:scimId(1)}], + meta:{resourceType:"Group",version:'W/"1"',created:"2026-10-09T00:00:00Z",lastModified:"2026-10-09T00:00:00Z",location:`/scim/v2/Groups/${scimId(3)}`} }; + assert.equal((await scimPut(f,group,null,0,"Groups")).status,200); + assert.equal(f.sql.prepare("SELECT team_id FROM tenant_memberships WHERE principal='user-1'").get().team_id,scimId(3)); + const foreign = {...group,id:scimId(4),meta:{...group.meta,location:`/scim/v2/Groups/${scimId(4)}`}}; + assert.equal((await scimPut(f,foreign,null,0,"Groups")).status,503); + assert.equal(f.sql.prepare("SELECT count(*) n FROM tenant_teams WHERE id=?").get(scimId(4)).n,0); +}); +test("workspace exact and semantic keys isolate two workspaces and preserve legacy bytes", async t => { + const f = fixture(t,{ORGANISATIONS_ENABLED:"true"}); + const retentionPolicy = {enabled:false}; + const ctx = context => ({retentionPolicy,tenantContext:context,cacheRevisions:{}}); + const exact = context => exactCacheIdentity(nativeRequest(),f.env,{...authority,tenantContext:context},ctx(context)); + const legacy = await exact(tenant()); + const old = await exactCacheIdentity(nativeRequest(),{...f.env,ORGANISATIONS_ENABLED:undefined},authority,ctx(tenant("org:a"))); + assert.deepEqual(legacy,old); + const one = await exact(tenant("org:a")),two = await exact(tenant("org:b")); + assert.notEqual(one.cache_key,two.cache_key); assert.notEqual(one.principal_hash,two.principal_hash); + const partition = (context, env = f.env) => semanticPartition("admin","openai",route,payload,ctx(context),{embedding_model:"openai:embed"},env); + assert.notDeepEqual(await partition(tenant("org:a")),await partition(tenant("org:b"))); + assert.deepEqual(await partition(tenant()),await partition(tenant("org:a"),{...f.env,ORGANISATIONS_ENABLED:undefined})); +}); +test("workspace idempotency claims cannot replay or hand off another workspace", async t => { + const f = fixture(t,{ORGANISATIONS_ENABLED:"true",MANAGED_IDEMPOTENCY_ENABLED:"true"}); + const body = {operation:"claim",scope:"a".repeat(64),digest:"b".repeat(64),owner:"c".repeat(32)}; + const call = async (data,context) => { + const response = await handleIdempotencyRequest(privateRequest("idempotency",data),f.env,{tenantContext:context}); + return {status:response.status,...await response.json()}; + }; + const one = await call(body,tenant("org:a")); assert.equal(one.result.status,"claimed"); + assert.equal((await call(body,tenant("org:b"))).result.status,"claimed"); + assert.equal((await call(body,tenant())).result.status,"claimed"); + assert.equal(f.sql.prepare("SELECT count(*) n FROM managed_idempotency").get().n,3); + assert.ok(f.sql.prepare("SELECT scope FROM managed_idempotency").all().some(row=>row.scope===body.scope)); +}); +test("workspace Responses owners and context page owners refuse foreign reads", async t => { + const f = fixture(t,{ORGANISATIONS_ENABLED:"true",HOSTED_RESPONSES_ENABLED:"true",CONTEXT_PAGING_ENABLED:"true"}); + const id = `gwresp_${"1".repeat(32)}`,owner = "a".repeat(64); + const put = {operation:"put",owner,id,provider:"openai",model:"test",parent_id:null,policy_revision:"b".repeat(64),depth:1, + document:{input:[{role:"user",content:"fixture"}],response:{id,object:"response",status:"completed",output:[]}}}; + const send = async (body,context) => { + const response = await handleResponsesStateRequest(privateRequest("responses",body),f.env,{tenantContext:context}); + return {status:response.status,...await response.json()}; + }; + assert.equal((await send(put,tenant("org:a"))).status,200); + assert.equal((await send({operation:"get",id,owner},tenant("org:a"))).result.state.document.response.id,id); + for (const other of [tenant("org:b"),tenant()]) assert.equal((await send({operation:"get",id,owner},other)).result.state,null); + const scope = {principal:"admin",session:"session-one",revision:"revision-one"}; + const group = [{role:"assistant",tool_calls:[{id:"call-one",type:"function",function:{name:"search",arguments:"{}"}}]}, + {role:"tool",tool_call_id:"call-one",content:"fixture"}]; + const page = async (body,context) => { + const response = await handleContextPageStateRequest(privateRequest("context-pages",{...body,granted:true,retention_policy:{enabled:false,mode:"inherit"}}),f.env,{tenantContext:context}); + return {status:response.status,...await response.json()}; + }; + const stored = await page({operation:"put",scope,bodies:[Buffer.from(JSON.stringify(group)).toString("base64")]},tenant("org:a")); + assert.equal(stored.status,200); const page_id = stored.pages[0].page_id; + assert.equal((await page({operation:"get",scope,page_id},tenant("org:a"))).status,200); + assert.equal((await page({operation:"get",scope,page_id},tenant("org:b"))).status,404); + assert.equal((await page({operation:"get",scope,page_id},tenant())).status,404); +}); +test("workspace usage attribution isolates rows and keeps legacy identity bytes", async t => { + const f = fixture(t,{ORGANISATIONS_ENABLED:"true"}); + for (const context of [tenant("org:a"),tenant("org:b"),tenant()]) await recordNativeUsage(f.env,{ + principal:"edge:fixture",tenantContext:context,requestId:crypto.randomUUID(),provider:"openai",model:"test",endpoint:route,status:200, + input_tokens:1,output_tokens:1,cost_usd:0.1,cost_basis:"usage",duration_ms:1,ttft_ms:1,outcome:"success", + },{}); + const principals = f.sql.prepare("SELECT principal FROM usage_events ORDER BY principal").all().map(row=>row.principal); + assert.deepEqual(principals.sort(),[tenantStorageKey("edge:fixture",tenant("org:a")),tenantStorageKey("edge:fixture",tenant("org:b")),"edge:fixture"].sort()); +}); +test("all flags off native streaming and account reads leave bytes and storage unchanged", async () => { + const forbidden = { prepare() { assert.fail("disabled storage"); } }; + const env = {INTELLIGENCE_DB:forbidden}; + for (const body of [completion, 'data: {"choices":[{"delta":{"content":"fixture"}}]}\n\ndata: [DONE]\n\n']) { + const response = await nativeGenerationFetch(nativeRequest(),env,{},authority,()=>new Response(body)); + assert.equal(await response.text(),body); assert.equal(response.headers.get("X-MultiLLM-Cache"),null); + } + const accountRequest = new Request("https://provider.invalid/v1/models"); + assert.equal(await (await nativeGenerationFetch(accountRequest,{...env,...allFlags},{},authority,()=>new Response("models"))).text(),"models"); +}); +test("all flags on forwarded webhook, SCIM and SAML retain raw bytes with no edge key or policies", async () => { + const {default:worker} = await loadWorkerModule(); + for (const path of ["/v1/payments/webhook","/scim/v2/Users","/auth/saml/acs","/v1/chat/completions"]) { + const raw = ' {"fixture":"é"}\r\n'; let calls=0; + const env = {...allFlags,INTELLIGENCE_DB:{prepare(){assert.fail("forwarded edge storage");}}, + MULTILLM_PROXY_CONTAINER:{getByName:()=>({async fetch(request){calls++;assert.equal(await request.text(),raw);return new Response("forwarded");}})}}; + const result = await worker.fetch(new Request(`https://gateway.example${path}`,{method:"POST",body:raw}),env,{waitUntil(){}}); + assert.equal(await result.text(),"forwarded"); assert.equal(calls,1); + } +}); + +test("concurrent cancellation finalizers release one immutable hold once", async t => { + const f = fixture(t,{CREDITS_ENABLED:"true",CREDITS_ENFORCEMENT:"all"}); await fund(f); + const hooks = createCreditsLifecycle(f.env,{context:tenant(),scoped_id:"concurrent_attempt"}); + await hooks.admit({amount:73,tariff_revision:7}); + await Promise.all([hooks.finalize({handedOff:false}),hooks.finalize({handedOff:false})]); + assert.equal((await readCredits(f.db,"admin")).held_microusd,0); + assert.equal(f.sql.prepare("SELECT count(*) n FROM credits_entries WHERE kind='release'").get().n,1); +}); +test("all flags on credit denial cancels the earlier governance hold and never leases admission", async t => { + const f = fixture(t,{...allFlags,USAGE_RESERVATIONS_ENABLED:"true",ADMISSION_ENABLED:"true", + ADMISSION_LIMITS_JSON:'{"principal":1}',ADMISSION_COORDINATOR:{getByName(){assert.fail("credit denial lease");}}}); + f.bind("org:a"); await fund(f,tenant("org:a"),0); + const response = await nativeGenerationFetch(nativeRequest(),f.env,{},authority,()=>assert.fail("credit denial upstream")); + assert.equal(response.status,402); + assert.equal(f.sql.prepare("SELECT state FROM tenant_governance_reservations").get().state,"settled"); + assert.equal(f.sql.prepare("SELECT charged FROM tenant_governance_reservations").get().charged,0); + assert.equal((await readCredits(f.db,contextOwner(tenant("org:a")))).held_microusd,0); +}); +test("each native attempt owns its hold and an undispatched loser releases only its estimate", async t => { + const f = fixture(t,{CREDITS_ENABLED:"true",CREDITS_ENFORCEMENT:"all"}); await fund(f); + const winner = createCreditsLifecycle(f.env,{context:tenant(),scoped_id:"winner"}); + const loser = createCreditsLifecycle(f.env,{context:tenant(),scoped_id:"loser"}); + await winner.admit({amount:73,tariff_revision:7}); await loser.admit({amount:73,tariff_revision:7}); + await winner.before_dispatch(); + await winner.finalize({outcome:"success",cost_basis:"usage",cost_usd:0.000007}); + await loser.finalize({handedOff:false}); + const summary = await readCredits(f.db,"admin"); + assert.equal(summary.balance_microusd,999993); assert.equal(summary.held_microusd,0); + assert.deepEqual(summary.entries.slice(1).map(row=>row.operation_id),["winner.reserve","loser.reserve","winner.commit","loser.release"]); +}); +test("uncertain credit settlement fails closed and preserves the dispatched hold", async t => { + const f = fixture(t,{CREDITS_ENABLED:"true",CREDITS_ENFORCEMENT:"all"}); await fund(f); + const hooks = createCreditsLifecycle(f.env,{context:tenant(),scoped_id:"uncertain_attempt"}); + await hooks.admit({amount:73,tariff_revision:7}); await hooks.before_dispatch(); + f.failNextBatch(); + await assert.rejects(hooks.finalize({outcome:"success",cost_basis:"usage",cost_usd:0.000007}),error=>error.code==="credits_unavailable"); + assert.equal((await readCredits(f.db,"admin")).held_microusd,73); +}); + +test("native semantic embeddings charge separate attempts and a hit releases the generation estimate", async t => { + const f = fixture(t,{...allFlags,SEMANTIC_CACHE_ENABLED:"true",SEMANTIC_CACHE_POLICY_JSON:JSON.stringify({ + routes:[route],embedding_model:"openai:embed"}), + MODEL_PRICING_USD_PER_MILLION:'{"openai:test":{"input":1,"output":2},"openai:embed":{"input":0.1,"output":0}}'}); + f.bind("org:a"); await fund(f,tenant("org:a")); const calls=[],waits=[]; + const fetcher = req => { calls.push(new URL(req.url).pathname); return req.url.endsWith("/embeddings") + ? Response.json({data:[{embedding:[1,0]}],usage:{prompt_tokens:3}}) : complete(); }; + for (let i=0;i<2;i++) { + const response = await nativeGenerationFetch(nativeRequest(),f.env,{waitUntil:p=>waits.push(p)},authority,fetcher); + assert.equal(await response.text(),completion); + if(i) assert.equal(response.headers.get("X-MultiLLM-Cache"),"semantic-hit"); + } + await Promise.all(waits); + assert.deepEqual(calls,["/test/v1/embeddings",route,"/test/v1/embeddings"]); + const summary = await readCredits(f.db,contextOwner(tenant("org:a"))); + assert.equal(summary.balance_microusd,999991); assert.equal(summary.held_microusd,0); + assert.equal(new Set(summary.entries.filter(row=>row.kind==="reserve").map(row=>row.scoped_id)).size,4); + assert.equal(summary.entries.filter(row=>row.kind==="release").length,1); +}); +test("Realtime disabled routes retain forwarding before every enabled enterprise policy", async () => { + const {default:worker} = await loadWorkerModule(); + for (const path of ["/v1/realtime/client_secrets"]) { + const env = {...allFlags,INTELLIGENCE_DB:{prepare(){assert.fail("realtime enterprise storage");}}, + MULTILLM_PROXY_CONTAINER:{getByName:()=>({fetch:async()=>new Response("realtime forwarded")})}}; + const response = await worker.fetch(new Request(`https://gateway.example${path}`,{method:"POST",body:"{}"}),env,{waitUntil(){}}); + assert.equal(response.status,200); assert.equal(await response.text(),"realtime forwarded"); + } +}); + +test("enabled governance leaves an unbound legacy request body and storage unchanged", async () => { + const raw="legacy opaque body"; + const request = new Request(`https://provider.invalid${route}`,{method:"POST",body:raw}); + const env={TENANT_GOVERNANCE_ENABLED:"true",ADMIN_API_KEY:"synthetic-admin-key",INTELLIGENCE_DB:{prepare(){assert.fail("legacy governance storage");}}}; + const response=await nativeGenerationFetch(request,env,{},authority,async req=>new Response(await req.text())); + assert.equal(await response.text(),raw); +}); +test("enabled workspace key grants use the same case-insensitive wildcard intersection", async t => { + const f=fixture(t,{ORGANISATIONS_ENABLED:"true",TENANT_GOVERNANCE_ENABLED:"true"}); f.bind("org:a"); + const response=await nativeGenerationFetch(nativeRequest(),f.env,{}, {...authority,principal:{id:"admin",allowed_models:"OPENAI:*es*"}},complete); + assert.equal(await response.text(),completion); +}); + +test("organisations off preserves claim and Responses ownership despite a supplied workspace context", async t => { + const f=fixture(t,{MANAGED_IDEMPOTENCY_ENABLED:"true",HOSTED_RESPONSES_ENABLED:"true"}); + const context=tenant("org:a"),scope="d".repeat(64),owner="e".repeat(64),id=`gwresp_${"2".repeat(32)}`; + const claimed=await handleManagedStateRequest(privateRequest("idempotency",{operation:"claim",scope,digest:"b".repeat(64),owner:"c".repeat(32)}),f.env,{tenantContext:context}); + assert.equal(claimed.status,200); assert.equal(f.sql.prepare("SELECT scope FROM managed_idempotency").get().scope,scope); + const saved=await handleManagedStateRequest(privateRequest("responses",{operation:"put",owner,id,provider:"openai",model:"test",parent_id:null, + policy_revision:"b".repeat(64),depth:1,document:{input:[{role:"user",content:"fixture"}],response:{id,object:"response",status:"completed",output:[]}}}),f.env,{tenantContext:context}); + assert.equal(saved.status,200); assert.equal(f.sql.prepare("SELECT owner FROM hosted_responses").get().owner,owner); +}); +test("repeated revision mismatches stop after three attempts with no dispatch", async () => { + let attempts=0; + const hooks=createCreditsLifecycle({CREDITS_ENABLED:"true",CREDITS_ENFORCEMENT:"all"},{context:tenant(), + read:async()=>({revision:attempts}),authority:()=>({reserve(){attempts++;throw Object.assign(Error(),{status:412});},commit(){},reconcile(){}})}); + await assert.rejects(hooks.admit({amount:73,tariff_revision:7}),error=>error.code==="credits_unavailable"); + assert.equal(attempts,3); +}); + +test("organisations off preserves native budget ownership with a supplied workspace context", async t => { + const f=fixture(t,{USAGE_RESERVATIONS_ENABLED:"true"}); + const response=await nativeGenerationFetch(nativeRequest(),f.env,{}, {...authority,tenantContext:tenant("org:a"), + principal:{id:"admin",daily_budget_usd:1}},complete); + assert.equal(await response.text(),completion); + assert.equal(f.sql.prepare("SELECT principal FROM usage_reservations").get().principal, + `edge:${await retentionRequestId("native-admin:admin")}`); +}); + +for (const workspace of [false, true]) test(`operator reconciliation settles linked native credits once; workspace=${workspace}`, async t => { + const f = fixture(t, { CREDITS_ENABLED: "true", CREDITS_ENFORCEMENT: "all", USAGE_RESERVATIONS_ENABLED: "true", + ...(workspace ? { ORGANISATIONS_ENABLED: "true", TENANT_GOVERNANCE_ENABLED: "true" } : {}) }); + if (workspace) f.bind("org:a"); + const context = tenant(workspace ? "org:a" : undefined); + await fund(f, context); + const checked = { ...authority, principal: { ...authority.principal, daily_budget_usd: 1 } }; + const response = await nativeGenerationFetch(nativeRequest(), f.env, {}, checked, + () => new Response(JSON.stringify({ choices: [{ message: { content: "synthetic" } }] }), { headers: { "content-type": "application/json" } })); + await response.text(); + const row = f.sql.prepare(`SELECT * FROM ${workspace ? "tenant_governance_reservations" : "usage_reservations"}`).get(); + assert.equal(row.state, "unknown"); + const balance = await readCredits(f.db, contextOwner(context)); + assert.equal(balance.entries.find(entry => entry.kind === "reserve" && entry.held_delta > 0).scoped_id, row.id); + const call = async body => { + const handler = workspace ? handleTenantGovernanceRequest : handleReservationsRequest; + const response = await handler(new Request(`http://intelligence.internal/v1/${workspace ? "tenant-governance" : "reservations"}`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }) }), f.env); + return { status: response.status, ...await response.json() }; + }; + const values = { id: row.id, revision: row.revision, transition_id: "b".repeat(32), admin: true, reason: "review", evidence: "receipt" }; + const body = workspace ? { operation: "reconcile", cost: 25, ...values } + : { operation: "transition", state: "reconciled", cost_usd: 0.000025, basis: "provider", settlement_id: row.id, ...values }; + for (const applied of [true, false]) { + const result = await call(body); + assert.equal(result.status, 200); assert.equal(result.applied, applied); + } + const settled = await readCredits(f.db, contextOwner(context)); + assert.equal(settled.balance_microusd, 999975); assert.equal(settled.held_microusd, 0); + assert.equal(settled.entries.filter(entry => entry.kind === "commit").length, 1); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 0b6d24fb..3f1c85b8 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -42,6 +42,7 @@ const DIRECT_ENV_KEYS = [ "CONFIG_SYNC_TTL_SECONDS", "CREDITS_CURRENCY", "CREDITS_ENABLED", + "CREDITS_ENFORCEMENT", "ORGANISATIONS_ENABLED", "PAYMENTS_ENABLED", "PAYMENT_PROCESSOR_CONFIG_JSON", diff --git a/worker/context-pages-d1.mjs b/worker/context-pages-d1.mjs index 75f33f3d..738ca782 100644 --- a/worker/context-pages-d1.mjs +++ b/worker/context-pages-d1.mjs @@ -1,3 +1,4 @@ +import { organisationsEnabled, resolveAccountTenant, tenantStorageKey } from "./tenants-d1.mjs"; /** Exact context bodies in R2, signed principal/session/revision handles in D1. */ import { RoleplayTurnError } from "./roleplay/turn-runtime.mjs"; import { mediaMac, mediaSecret, toBase64, toBase64Url } from "./media-signing.mjs"; @@ -60,7 +61,8 @@ async function roleplayScope(env, keyScope, session, policy) { env.PII_REDACTION_ENABLED ?? "", env.PII_REDACTION_POLICY_JSON ?? "", env.PROMPT_INJECTION_MODE ?? "", env.PROMPT_INJECTION_THRESHOLD ?? "", env.PROMPT_INJECTION_POLICY_JSON ?? "", env.SECRET_SCAN_ENABLED ?? "", env.SECRET_SCAN_DEFAULT ?? "", env.CONTEXT_PAGING_ENABLED ?? ""]))); - return { principal: keyScope, session, revision }; + const context = organisationsEnabled(env) ? await resolveAccountTenant(env.INTELLIGENCE_DB, env.ADMIN_USERNAME || "admin", env) : null; + return { principal: tenantStorageKey(keyScope, context), session, revision }; } /** Attach verified identity only after authentication, on the private DO hop. */ @@ -322,7 +324,7 @@ export async function handleContextPageRequest(request, env, { authorize, store } /** Private Container calls are reachable only through the authenticated state dispatcher. */ -export async function handleContextPageStateRequest(request, env) { +export async function handleContextPageStateRequest(request, env, { tenantContext } = {}) { const url = new URL(request.url); if (url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/managed-state/context-pages" || url.search || url.hash || url.username || url.password || request.method !== "POST") return failure(new ContextPageError("invalid_store_target", 400)); @@ -346,6 +348,9 @@ export async function handleContextPageStateRequest(request, env) { let offset = 0; for (const chunk of chunks) { input.set(chunk, offset); offset += chunk.length; } const body = JSON.parse(decoder.decode(input)); + if (organisationsEnabled(env) && tenantContext && body.scope) body.scope = { + ...body.scope, principal: tenantStorageKey(body.scope.principal, tenantContext), + }; const policy = retentionPolicySnapshot(body.retention_policy); if (body.granted !== true || !retentionAllowsContent(policy)) throw new ContextPageError("context_page_forbidden", 403); const store = new ContextPageStore(env); diff --git a/worker/control-users-d1.mjs b/worker/control-users-d1.mjs index 363ef2b7..530d7d62 100644 --- a/worker/control-users-d1.mjs +++ b/worker/control-users-d1.mjs @@ -305,3 +305,15 @@ export async function handleControlUsersRequest(request, env) { return reply({ error: "storage_unavailable" }, 503); } } + +/** Statements only: the SCIM authority commits accounts and identities in one batch. */ +export function scimAccountStatements(db, body, priorAccount, guard) { + if (!body.account) return []; + const { account, expected } = body; + if (expected === 0) return [db.prepare(`INSERT INTO control_users (${USER_FIELDS.join(",")}) VALUES (${USER_FIELDS.map(() => "?").join(",")})`) + .bind(...USER_FIELDS.map(key => account[key]))]; + return [db.prepare(`UPDATE control_users SET api_key_hash=?,api_key_prefix=?,revoked_at=? + WHERE username=? AND is_admin=0 AND api_key_hash=? AND api_key_prefix=? AND revoked_at IS ?`) + .bind(account.api_key_hash, account.api_key_prefix, account.revoked_at, account.username, + priorAccount.api_key_hash, priorAccount.api_key_prefix, priorAccount.revoked_at), guard()]; +} diff --git a/worker/cors-policy.mjs b/worker/cors-policy.mjs index 9e98d5e0..5ed11084 100644 --- a/worker/cors-policy.mjs +++ b/worker/cors-policy.mjs @@ -1,6 +1,6 @@ export const CORS_ALLOWED_METHODS = "GET, POST, PUT, DELETE, PATCH, OPTIONS"; export const CORS_DEFAULT_HEADERS = - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, X-MultiLLM-Priority, Prefer, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Report, X-MultiLLM-Retention, X-MultiLLM-Deadline-Ms, If-Match, Authorization, X-Api-Key, X-Goog-Api-Key, X-MultiLLM-Api-Key, X-Roleplay-Session-ID, Anthropic-Version, Anthropic-Beta, Anthropic-Dangerous-Direct-Browser-Access, Content-Type, Accept, Origin, X-Requested-With, OpenAI-Beta, OpenAI-Organization, OpenAI-Project, Idempotency-Key, X-MultiLLM-Priority, Prefer, Moderation, Moderation-Model, Redaction, X-Client-Request-ID, X-App-Name, X-Billing-Mode, X-BYOK-Provider, X-Encryption-Key, X-Encryption-Passphrase, X-Fal-Object-Lifecycle-Preference, X-PAYMENT, X-Prompt-Caching-Cut-After, X-Provider, X-Team-ID, X-Use-BYOK, x-x402"; export const CORS_EXPOSE_HEADERS = "X-MultiLLM-Secret-Scan, X-MultiLLM-Canary-Cohort, X-MultiLLM-Injection-Action, X-MultiLLM-Retention, X-MultiLLM-Roleplay-Recovery, " + - "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, Location, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; + "X-MultiLLM-Tool-Repair, X-MultiLLM-Image-QA, X-MultiLLM-Cascade, X-MultiLLM-Conversion-Fidelity, X-MultiLLM-Conversion-Fields, Retry-After, Location, ETag, X-Request-ID, X-MultiLLM-RateLimit-Limit, X-MultiLLM-RateLimit-Remaining, X-MultiLLM-RateLimit-Reset, X-MultiLLM-Idempotency, X-MultiLLM-Optimization, X-MultiLLM-Optimization-Mode, X-MultiLLM-Estimated-Input-Before, X-MultiLLM-Estimated-Input-After, X-MultiLLM-Image-Prompts-Compacted, X-MultiLLM-Messages-Summarized, X-MultiLLM-Optimization-Target-Met, X-MultiLLM-Summary, X-MultiLLM-Optimization-Cache-Hits, X-MultiLLM-Optimization-Cache-Misses, X-MultiLLM-Prompt-Cache, X-MultiLLM-Prompt-Cache-Mode, X-MultiLLM-Prompt-Cache-Estimated-Tokens, X-MultiLLM-Provider, X-MultiLLM-Model, X-MultiLLM-Credential-Attempts, X-MultiLLM-Route-Decision, X-MultiLLM-Circuit-State, X-MultiLLM-Auto-Route, X-MultiLLM-Auto-Selected-Model, X-MultiLLM-Auto-Attempts, X-MultiLLM-Auto-Selected-Priority, X-MultiLLM-Auto-Ordering, X-MultiLLM-Auto-Failover-Reasons, X-MultiLLM-Cache, X-MultiLLM-Cache-Backend, X-MultiLLM-Usage-Basis, X-MultiLLM-Provider-Calls, Age, X-MultiLLM-Transport-Failure, X-MultiLLM-Token-Count, X-MultiLLM-Latency-Ms, X-MultiLLM-Estimated-Cost-USD, X-MultiLLM-Cost-Basis, X-MultiLLM-Reasoning-Normalized, X-Roleplay-Session-ID, X-Roleplay-Trace-ID, X-Roleplay-Session-Source, X-Roleplay-Provider, X-Roleplay-Model, X-Roleplay-Selection, X-Roleplay-Memory, X-Roleplay-Estimated-Input-Tokens, X-Roleplay-Max-Output-Tokens, X-Roleplay-Fallback-Count, X-Roleplay-State-Cache, X-Roleplay-Credential-Check, Server-Timing, WWW-Authenticate, X-PAYMENT-RESPONSE, X-Poll-After, X-NanoGPT-Advisor-ID, X-NanoGPT-Data-Endpoint, X-NanoGPT-Direct-Endpoint, X-NanoGPT-Inline-Moderation-Cost-USD, X-NanoGPT-Inline-Moderation-Flagged, X-NanoGPT-Inline-Moderation-Model"; diff --git a/worker/credits-admission.mjs b/worker/credits-admission.mjs new file mode 100644 index 00000000..152ac981 --- /dev/null +++ b/worker/credits-admission.mjs @@ -0,0 +1,117 @@ +/** Per-attempt native charging, using the immutable enterprise credit authority. */ +import { createHash } from "node:crypto"; +import { creditsEnabled, creditsEnforcement, contextOwner, readCredits, appendCredit, registerCreditAuthority } from "./credits-d1.mjs"; +import { TenantContext, AuthorityOperation, callAuthority, registerEnterpriseAdapters } from "./enterprise-contract.mjs"; + +export class CreditsAdmissionError extends Error { + constructor(code = "credits_unavailable", status = 503) { super(code); this.code = code; this.status = status; } + response() { return Response.json({ error: { code: this.code, message: "Credit admission could not be completed." } }, { status: this.status }); } +} +export function integerMicroUsd(value) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new CreditsAdmissionError("credits_unpriced"); + const [mantissa, exponent = "0"] = String(value).toLowerCase().split("e"); + const [whole, fraction = ""] = mantissa.split("."); + const digits = BigInt(whole + fraction), power = 6 + Number(exponent) - fraction.length; + const divisor = power < 0 ? 10n ** BigInt(-power) : 1n; + const result = power < 0 ? (digits + divisor - 1n) / divisor : digits * 10n ** BigInt(power); + if (result > BigInt(Number.MAX_SAFE_INTEGER - 1)) throw new CreditsAdmissionError("credits_unpriced"); + return Number(result); +} +export function nativeTariffRevision(env) { + const raw = env.MODEL_PRICING_USD_PER_MILLION; + if (typeof raw !== "string" || !raw || raw.length > 65536) return null; + try { const table = JSON.parse(raw); if (!table || Array.isArray(table) || typeof table !== "object") return null; } + catch { return null; } + return Number.parseInt(createHash("sha256").update(raw).digest("hex").slice(0, 12), 16); +} +/** Only failed compare-and-swap operations retry; IDs and amounts never change. */ +export async function withCreditRevision(db, owner, apply, read = readCredits) { + for (let attempt = 0; attempt < 3; attempt++) { + const revision = (await read(db, owner)).revision; + try { return await apply(revision); } + catch (error) { if (error.status !== 412 || attempt === 2) throw error; } + } +} +function admissionFailure(error, phase) { + if (error instanceof CreditsAdmissionError) return error; + if (phase === "reserve" && error.code === "credits_insufficient") return new CreditsAdmissionError("credits_insufficient", 402); + return new CreditsAdmissionError(error.code === "credits_unpriced" ? "credits_unpriced" : "credits_unavailable"); +} +export function createCreditsLifecycle(env, { context, db = env.INTELLIGENCE_DB, scoped_id, + read = readCredits, append = appendCredit, authority = registerCreditAuthority } = {}) { + const mode = creditsEnforcement(env); + scoped_id ??= mode === "off" ? "disabled" : `native_${crypto.randomUUID().replaceAll("-", "")}`; + let owner, charging, held = false, dispatched = false, finished = false, estimate, revision, finalEvent; + + const decide = async () => { + if (mode === "off") return false; + if (charging !== undefined) return charging; + try { + owner = contextOwner(context); + charging = mode === "all" || (await read(db, owner)).revision > 0; + return charging; + } catch (error) { throw admissionFailure(error, "reserve"); } + }; + const apply = async (phase, amount) => { + const adapters = registerEnterpriseAdapters({ credit: authority(db, { env, + tariff: (operation, action) => action === "reconcile" || operation.amount !== amount + || operation.scoped_id !== scoped_id ? null : revision }) }); + return withCreditRevision(db, owner, current => callAuthority(adapters, "credit", phase, + new AuthorityOperation({ context, scoped_id, revision: current, operation_id: `${scoped_id}.${phase}`, amount })), read); + }; + const hooks = { + requiresPricing: decide, + async admit(event) { + if (held || finished || !await decide()) return; + if (event.scoped_id !== undefined) scoped_id = event.scoped_id; + estimate = event.amount; revision = event.tariff_revision; + if (!Number.isSafeInteger(estimate) || estimate < 0 || !Number.isSafeInteger(revision) || revision < 0) + throw new CreditsAdmissionError("credits_unpriced"); + try { await apply("reserve", estimate); held = true; } + catch (error) { throw admissionFailure(error, "reserve"); } + if (finalEvent) await hooks.finalize(finalEvent); + }, + async before_dispatch() { if (held && !finished) dispatched = true; }, + async finalize(event) { + finalEvent = event; + if (!held || finished) return; + const row = event.usage ?? event; + try { + if (!dispatched || event.handedOff === false || row.cost_basis === "cache") { + await withCreditRevision(db, owner, current => append(db, { owner, kind: "release", amount_microusd: estimate, + scoped_id, revision: current, operation_id: `${scoped_id}.release` }), read); + } else if ((!event.outcome || event.outcome === "success") && !event.cancellationOutcome?.ambiguous && row.cost_basis === "usage" + && (Number.isSafeInteger(row.cost_micro_usd) || row.cost_usd != null)) { + await apply("commit", row.cost_micro_usd ?? integerMicroUsd(row.cost_usd)); + } else await apply("reconcile", 0); + finished = true; + } catch (error) { throw admissionFailure(error, "settle"); } + }, + }; + return hooks; +} + +/** Recover a linked attempt from immutable ledger evidence, without current membership. */ +export async function reconcileCreditsReservation(env, row, transition_id) { + if (!creditsEnabled(env) || row.cost_usd == null) return; + const db = env.INTELLIGENCE_DB; + try { + const found = await db.prepare("SELECT owner,kind,amount_microusd,held_delta,tariff_revision FROM credits_entries WHERE scoped_id=?") + .bind(row.id).all(); + const entries = found.results; + const reserve = entries.find(entry => entry.kind === "reserve" && entry.held_delta > 0); + if (!reserve) return; + if (entries.some(entry => entry.owner !== reserve.owner)) throw new CreditsAdmissionError(); + if (entries.some(entry => ["commit", "release"].includes(entry.kind))) return; + let principal_id = reserve.owner, org_id = null, team_id = null; + if (principal_id.startsWith("tenant:")) [org_id, team_id, principal_id] = JSON.parse(principal_id.slice(7)); + const context = new TenantContext({ principal_id, org_id, team_id }); + const amount = integerMicroUsd(row.cost_usd); + const operation_id = createHash("sha256").update(JSON.stringify([row.id, "reconcile", transition_id])).digest("hex"); + const adapters = registerEnterpriseAdapters({ credit: registerCreditAuthority(db, { env, + tariff: (operation, phase) => phase === "reconcile" && operation.amount === amount + && operation.scoped_id === row.id ? reserve.tariff_revision : null }) }); + await withCreditRevision(db, reserve.owner, revision => callAuthority(adapters, "credit", "reconcile", + new AuthorityOperation({ context, scoped_id: row.id, revision, operation_id, amount }))); + } catch (error) { throw admissionFailure(error, "settle"); } +} diff --git a/worker/credits-d1.mjs b/worker/credits-d1.mjs index 7e1f0588..0e80098e 100644 --- a/worker/credits-d1.mjs +++ b/worker/credits-d1.mjs @@ -36,6 +36,18 @@ export function creditsEnabled(env = {}, warn = message => console.warn(message) return true; } +export function creditsEnforcement(env = {}, warn = message => console.warn(message)) { + const raw = env.CREDITS_ENFORCEMENT; + let mode = raw === undefined ? '' : typeof raw === 'string' ? raw.trim().toLowerCase() : 'invalid'; + if (!['', 'off', 'funded', 'all'].includes(mode)) { + if (!warned.has('CREDITS_ENFORCEMENT')) { + warned.add('CREDITS_ENFORCEMENT'); warn('Invalid CREDITS_ENFORCEMENT; credits disabled'); + } + mode = 'off'; + } + return creditsEnabled(env, warn) ? mode || 'off' : 'off'; +} + function integer(value, nonnegative = false) { if (!Number.isSafeInteger(value) || Math.abs(value) > MAX_AMOUNT || nonnegative && value < 0) { throw new CreditsError('invalid_credits_request', 400); @@ -73,6 +85,7 @@ function requestFields(data) { throw new CreditsError('invalid_credits_request', 400); } for (const value of [scoped_id, reference, actor]) if (value !== null) label(value); + if (actor?.startsWith('processor:')) throw new CreditsError('invalid_credits_request', 400); if (['reserve', 'commit', 'release'].includes(kind)) { label(scoped_id); integer(amount_microusd, true); } if (['reserve', 'commit'].includes(kind) && !unknown) { if (tariff_revision === null) throw new CreditsError('credits_unpriced'); @@ -128,14 +141,34 @@ async function schema(db) { } export async function appendCredit(db, data) { - const fields = requestFields(data); + return appendFields(db, requestFields(data)); +} + +export async function appendPaymentCredit(db, data) { + const { owner, event_kind, amount_microusd, operation_id, revision, scoped_id, processor_id } = data; + if (!['credit', 'refund', 'dispute'].includes(event_kind) || processor_id !== 'stripe') { + throw new CreditsError('invalid_credits_request', 400); + } + integer(amount_microusd, true); label(scoped_id); + const fields = requestFields({ owner, kind: event_kind === 'credit' ? 'credit' : 'adjust', + amount_microusd: event_kind === 'credit' ? amount_microusd : -amount_microusd, + operation_id, revision, scoped_id, actor: processor_id, reason: event_kind }); + fields.actor = `processor:${processor_id}`; + return appendFields(db, fields, true); +} + +async function appendFields(db, fields, payment = false) { const document = documentFor(fields); try { await schema(db); const find = () => db.prepare('SELECT * FROM credits_entries WHERE owner=? AND operation_id=?') .bind(fields.owner, fields.operation_id).first(); const replay = row => { - if (row.document !== document) throw new CreditsError('credits_conflict', 409); + if (row.document !== document) { + const prior = JSON.parse(row.document), current = JSON.parse(document); + delete prior.revision; delete current.revision; + if (!payment || documentFor(prior) !== documentFor(current)) throw new CreditsError('credits_conflict', 409); + } return publicEntry(row); }; const existing = await find(); @@ -153,7 +186,7 @@ export async function appendCredit(db, data) { integer(balance + bd); integer(held + hd, true); integer(revision + 1, true); const available = balance + bd - held - hd; const reservation = fields.kind === 'reserve' && !fields.unknown; - const enforcedSpend = reservation || (fields.kind === 'adjust' && fields.amount_microusd < 0); + const enforcedSpend = reservation || (fields.kind === 'adjust' && fields.amount_microusd < 0 && !payment); if (enforcedSpend && (available < 0 || (reservation && balance - held <= 0))) { throw new CreditsError('credits_insufficient', 409); } @@ -236,15 +269,17 @@ export async function handleCreditsRequest(request, env) { try { if (request.method !== 'POST') return reply({ error: { code: 'method_not_allowed' } }, 405); const body = await boundedBody(request); - if (!body || typeof body !== 'object' || Array.isArray(body) || body.version !== 1 || !['append', 'read'].includes(body.operation)) { + if (!body || typeof body !== 'object' || Array.isArray(body) || body.version !== 1 || !['append', 'payment_append', 'read'].includes(body.operation)) { throw new CreditsError('invalid_credits_request', 400); } - const required = body.operation === 'append' ? ['owner', 'kind', 'amount_microusd', 'operation_id', 'revision'] : ['owner']; - const optional = body.operation === 'append' ? ['scoped_id', 'reference', 'tariff_revision', 'actor', 'reason', 'unknown'] : ['cursor', 'limit']; + const required = body.operation === 'payment_append' ? ['owner', 'event_kind', 'amount_microusd', 'operation_id', 'revision', 'scoped_id', 'processor_id'] + : body.operation === 'append' ? ['owner', 'kind', 'amount_microusd', 'operation_id', 'revision'] : ['owner']; + const optional = body.operation === 'payment_append' ? [] : body.operation === 'append' ? ['scoped_id', 'reference', 'tariff_revision', 'actor', 'reason', 'unknown'] : ['cursor', 'limit']; if (required.some(key => !(key in body)) || Object.keys(body).some(key => !['version', 'operation', ...required, ...optional].includes(key))) { throw new CreditsError('invalid_credits_request', 400); } const db = env.INTELLIGENCE_DB; + if (body.operation === 'payment_append') return reply({ entry: await appendPaymentCredit(db, body) }); return body.operation === 'append' ? reply({ entry: await appendCredit(db, body) }) : reply({ summary: await readCredits(db, body.owner, body) }); } catch (error) { diff --git a/worker/exact-generation-cache.mjs b/worker/exact-generation-cache.mjs index 49e6b104..0e958c6f 100644 --- a/worker/exact-generation-cache.mjs +++ b/worker/exact-generation-cache.mjs @@ -1,3 +1,4 @@ +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; /** Opt-in exact Chat Completions caching after native authentication and retention. */ import { GenerationCacheD1, MAX_BODY_BYTES, digest, completeCacheBody } from "./generation-cache-d1.mjs"; import { createUsageObserver } from "./request-telemetry.mjs"; @@ -54,7 +55,8 @@ export async function exactCacheIdentity(request, env, authority, context) { let payload; try {payload = JSON.parse(new TextDecoder("utf-8", {fatal: true}).decode(bytes));} catch {return null;} if (!eligibleRequest(payload)) return null; - const principal = await digest(`${authority.principal.id}\0${await digest(env.ADMIN_API_KEY)}`); + const principalId = organisationsEnabled(env) ? tenantStorageKey(authority.principal.id, authority.tenantContext ?? context.tenantContext) : authority.principal.id; + const principal = await digest(`${principalId}\0${await digest(env.ADMIN_API_KEY)}`); const policyHash = await nativePolicyHash(request, env, authority, context); const key = await digest(JSON.stringify([principal, authority.route, policyHash, canonical(payload)])); return {principal_hash: principal, cache_key: key, policy_hash: policyHash, model: payload.model}; diff --git a/worker/gateway-extensions.mjs b/worker/gateway-extensions.mjs index 59a47807..93c39ad0 100644 --- a/worker/gateway-extensions.mjs +++ b/worker/gateway-extensions.mjs @@ -1,3 +1,6 @@ +import { governanceEnabled, GovernanceError } from "./tenant-governance-d1.mjs"; +import { creditsEnforcement } from "./credits-d1.mjs"; +import { CreditsAdmissionError } from "./credits-admission.mjs"; /** Static registration boundary for native dispatch; later integrations supply named hooks. */ import { organisationsEnabled, resolveAccountTenant, tenantNamespace, TenantError } from "./tenants-d1.mjs"; import { generationCacheSettings, prepareNativeCache, cacheServedEvent } from "./exact-generation-cache.mjs"; @@ -73,7 +76,7 @@ async function nativeContext(request, env, authority, metrics) { .map(([name, state]) => [name, state.revision])) : {}; return Object.freeze({ provider: authority.provider, principal, model: typeof model === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/+@-]{0,223}$/.test(model) ? model : null, - ...(organisationsEnabled(env) ? { tenantContext: authority.tenantContext, tenantNamespace: tenantNamespace(authority.tenantContext) } : {}), + ...((organisationsEnabled(env) || authority.tenantContext) ? { tenantContext: authority.tenantContext, tenantNamespace: tenantNamespace(authority.tenantContext) } : {}), modelGroup: admissionModelGroup(authority.provider, model), requestId: crypto.randomUUID(), endpoint: authority.route, startedAt, retentionPolicy, cacheRevisions }); } @@ -84,8 +87,10 @@ class NativeGenerationRejection extends Error { } export function generationErrorResponse(error) { + if (error instanceof GovernanceError) return Response.json({ error: { code: error.code, + message: "Workspace governance denied generation", ...error.details } }, { status: error.status }); return error instanceof GenerationDeadlineExceeded || error instanceof InvalidGenerationDeadline - || error instanceof ReservationError || error instanceof AdmissionError || error instanceof NativeGenerationRejection ? error.response() : null; + || error instanceof CreditsAdmissionError || error instanceof ReservationError || error instanceof AdmissionError || error instanceof NativeGenerationRejection ? error.response() : null; } export function forwardedGenerationHeaders(request, env, deadline = createGenerationDeadline(request, env)) { @@ -279,7 +284,7 @@ async function runNativeLifecycle(request, env, ctx, authority, fetcher, registr } policies = await prepareNativePolicies(request, env, authority, context, settings, hook, race); request = policies.request; authority = policies.authority; context = policies.context; - reservation = settings.reservations ? nativeReservationLifecycle(policies.dispatchRequest, env, authority) : null; + reservation = (settings.reservations || settings.enterprise) ? nativeReservationLifecycle(policies.dispatchRequest, env, authority) : null; if (["baseline", "candidate"].includes(authority.canary?.cohort) && ["shadow", "live"].includes(authority.canary?.mode)) { context = { ...context, canary: { cohort: authority.canary.cohort, mode: authority.canary.mode } }; @@ -287,8 +292,10 @@ async function runNativeLifecycle(request, env, ctx, authority, fetcher, registr ctx?.waitUntil?.(new Promise(resolve => { resolveDone = resolve; })); lifecycle = createGatewayLifecycle([...registrations, ...nativeFinalizers(env, ctx, settings, reservation, () => { hook.deadline?.stop(); resolveDone?.(); })]); + await race(() => reservation?.authorize?.(context)); await race(() => lifecycle.authorize(context)); await race(() => lifecycle.admit(context)); + if (settings.enterprise) await race(() => reservation.admit(context)); const state = { ...policies, lifecycle: { ...lifecycle, finalize: finish }, context, owner, hook, reservation, cacheEnabled: settings.cache, semanticEnabled: settings.semantic, race, ctx, accounting }; const dispatch = () => dispatchWithNativeHooks(request, env, authority, fetcher, state); @@ -315,6 +322,7 @@ async function runNativeLifecycle(request, env, ctx, authority, fetcher, registr await owner.close("interrupted"); const rejection = error instanceof ContextCanaryError ? Response.json({ error: { code: error.code, message: "Managed response inspection stopped generation" } }, { status: 502 }) : generationErrorResponse(error); + if (!lifecycle && reservation) await reservation.finalize({ ...context, handedOff: false }); if (lifecycle) await finish({ ...context, status: rejection?.status ?? (request.signal.aborted ? 499 : 502), outcome: request.signal.aborted ? "canceled" : "transport_error", input_tokens: null, output_tokens: null, duration_ms: Math.max(0, Math.round(performance.now() - context.startedAt)), ttft_ms: null, cost_usd: null, cost_basis: null }); @@ -324,7 +332,10 @@ async function runNativeLifecycle(request, env, ctx, authority, fetcher, registr } export async function nativeGenerationFetch(request, env, ctx, authority, fetcher, collaborators = []) { - if (organisationsEnabled(env) && authority.authenticated === true) { + const path = authority.route ?? new URL(request.url).pathname; + if (!generationPath(request, path)) return fetcher(request, env, authority); + let enterprise = authority.authenticated === true && (governanceEnabled(env) || creditsEnforcement(env) !== "off"); + if ((organisationsEnabled(env) || enterprise) && authority.authenticated === true) { try { const tenantContext = await resolveAccountTenant(env.INTELLIGENCE_DB, authority.principal?.id ?? authority.keyId, env); authority = { ...authority, tenantContext, tenantNamespace: tenantNamespace(tenantContext) }; @@ -334,14 +345,14 @@ export async function nativeGenerationFetch(request, env, ctx, authority, fetche throw error; } } - const path = authority.route ?? new URL(request.url).pathname; - if (!generationPath(request, path)) return fetcher(request, env, authority); + enterprise = authority.authenticated === true && (creditsEnforcement(env) !== "off" + || governanceEnabled(env) && authority.tenantContext?.org_id != null); // The route has already authenticated the bootstrap key. Only public route metadata is trusted. const revision = nativeRevisionConsumer(env); tickNativeRevisionSync(env, ctx); const rejection = revision?.requireFreshSecurity(); if (rejection) { authority.deadlineHook?.deadline?.stop(); return rejection; } - const settings = { metrics: nativeMetricsEnabled(env), admission: admissionSettings(env).enabled, + const settings = { enterprise, metrics: nativeMetricsEnabled(env), admission: admissionSettings(env).enabled, cache: generationCacheSettings(env).enabled, reservations: reservationSettings(env).enabled, semantic: semanticCacheSettings(env).enabled, observability: observabilityEnabled(env), injection: resolveInjectionPolicy(env).mode !== "off", @@ -358,7 +369,7 @@ export async function nativeGenerationFetch(request, env, ctx, authority, fetche catch (error) { const response = generationErrorResponse(error); if (response) return response; throw error; } if (authority.onCancellationOutcome) hook.owner.onOutcome = authority.onCancellationOutcome; if (!settings.metrics && !settings.admission && !revision && !resolveRetentionPolicy(env).enabled - && !registrations.length && !settings.cache && !settings.reservations && !settings.semantic && !settings.injection && !settings.latency && !settings.streamCost && !settings.canary && !hook.deadline) { + && !registrations.length && !settings.cache && !settings.reservations && !settings.enterprise && !settings.semantic && !settings.injection && !settings.latency && !settings.streamCost && !settings.canary && !hook.deadline) { return dispatchNative(request, env, authority, fetcher, hook.owner); } return runNativeLifecycle(request, env, ctx, authority, fetcher, registrations, settings, hook); diff --git a/worker/idempotency-d1.mjs b/worker/idempotency-d1.mjs index cafcbad1..5cc82a03 100644 --- a/worker/idempotency-d1.mjs +++ b/worker/idempotency-d1.mjs @@ -1,3 +1,4 @@ +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; /** Atomic managed request claims and bounded complete response objects. */ import { boundedBody } from "./control-users-d1.mjs"; @@ -101,7 +102,7 @@ async function complete(db, bucket, body, now) { return { changed: changed.meta.changes === 1 }; } -export async function handleIdempotencyRequest(request, env) { +export async function handleIdempotencyRequest(request, env, { tenantContext } = {}) { const url = new URL(request.url); if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/managed-state/idempotency" || url.search || url.hash || url.username || url.password) return reply("not_found", 404); @@ -112,6 +113,7 @@ export async function handleIdempotencyRequest(request, env) { body = JSON.parse(await boundedBody(request, MAX_DOCUMENT_BYTES)); if (!validBody(body)) throw new Error(); } catch { return reply("invalid_idempotency_request", 400); } + if (organisationsEnabled(env) && tenantContext && body.scope) body.scope = tenantStorageKey(body.scope, tenantContext); const db = env.INTELLIGENCE_DB, bucket = env.multillm_media; if (!db || !bucket) return reply("idempotency_store_unavailable", 503); const now = Math.floor(Date.now() / 1000); diff --git a/worker/intelligence-outbound.mjs b/worker/intelligence-outbound.mjs index 8c324469..0a51a99b 100644 --- a/worker/intelligence-outbound.mjs +++ b/worker/intelligence-outbound.mjs @@ -1,3 +1,4 @@ +import { handleTenantGovernanceRequest } from "./tenant-governance-d1.mjs"; import { handleAdmissionRequest } from "./admission-do.mjs"; import { handleIntelligenceStoreRequest } from "./intelligence-d1.mjs"; import { handleIntelligenceAuthRequest } from "./intelligence-auth-d1.mjs"; @@ -25,6 +26,7 @@ export function handleIntelligenceOutbound(request, env) { return Response.json({ error: { code: "method_not_allowed", message: "Use POST for storage operations." } }, { status: 405, headers: { Allow: "POST" } }); } + if (url.pathname === "/v1/tenant-governance") return handleTenantGovernanceRequest(request, env); if (url.pathname === "/v1/reservations") return handleReservationsRequest(request, env); if (url.pathname === "/v1/gateway-batches") return handleBatchRequest(request, env); if (url.pathname === "/v1/admission") return handleAdmissionRequest(request, env); diff --git a/worker/knowledge-edge.mjs b/worker/knowledge-edge.mjs index 8ea18665..d5863ea9 100644 --- a/worker/knowledge-edge.mjs +++ b/worker/knowledge-edge.mjs @@ -276,7 +276,8 @@ async function dispatch(env, operation, principal, payload, signal) { throw error; } if (decision.header) principal.secretScanHeader = decision.header; - const body = JSON.stringify({ version: 1, operation, principal: { id: principal.id, scopes: principal.scopes }, payload, + const body = JSON.stringify({ version: 1, operation, principal: { id: principal.id, scopes: principal.scopes, + ...(organisationsEnabled(env) && principal.tenant_context ? { tenant_context: principal.tenant_context } : {}) }, payload, secret_scan_mode: decision.mode, secret_scan_checked: true, ...(principal.retentionPolicy?.enabled ? { retention_policy: principal.retentionPolicy } : {}) }); if (Buffer.byteLength(body) > (operation === "skills.sync" ? SYNC_REQUEST_BYTES : MAX_REQUEST_BYTES)) { diff --git a/worker/managed-state-dispatch.mjs b/worker/managed-state-dispatch.mjs index 66f6ee16..52cccd65 100644 --- a/worker/managed-state-dispatch.mjs +++ b/worker/managed-state-dispatch.mjs @@ -1,20 +1,33 @@ /** Fixed private managed-state domains; sibling authorities are injected explicitly. */ -import { handleTenantRequest, TENANT_STORE_PATH } from "./tenants-d1.mjs"; +import { handleTenantRequest, TENANT_STORE_PATH, scimPrincipalStatements, scimTeamStatements } from "./tenants-d1.mjs"; +import { handleSamlIdentityRequest } from "./saml-identity.mjs"; +import { handleScimStateRequest } from "./scim-d1.mjs"; +import { handleCreditsRequest } from "./credits-d1.mjs"; +import { handlePaymentStoreRequest } from "./payment-webhooks.mjs"; import { handleIdempotencyRequest } from "./idempotency-d1.mjs"; import { handleResponsesStateRequest } from "./responses-state-d1.mjs"; import { handleContextPageStateRequest } from "./context-pages-d1.mjs"; import { handleUsageReceiptStoreRequest } from "./usage-receipts.mjs"; -export function handleManagedStateRequest(request, env, { reservations, toolGrants } = {}) { +export function handleManagedStateRequest(request, env, { reservations, toolGrants, tenantContext } = {}) { const url = new URL(request.url); if (url.pathname === TENANT_STORE_PATH) return handleTenantRequest(request, env); if (!url.pathname.startsWith("/v1/managed-state/")) return null; if (url.origin !== "http://intelligence.internal" || url.search || url.hash || url.username || url.password || request.method !== "POST") { return Response.json({ version: 1, error: { code: "invalid_store_target", message: "Invalid managed storage target." } }, { status: 400 }); } - if (url.pathname === "/v1/managed-state/idempotency") return handleIdempotencyRequest(request, env); - if (url.pathname === "/v1/managed-state/responses") return handleResponsesStateRequest(request, env); - if (url.pathname === "/v1/managed-state/context-pages") return handleContextPageStateRequest(request, env); + if (url.pathname === "/v1/managed-state/saml") return handleSamlIdentityRequest(request, env); + if (url.pathname === "/v1/managed-state/scim") return handleScimStateRequest(request, env, { + adminUsername: env.ADMIN_USERNAME || "admin", + principalStatements: data => scimPrincipalStatements(env.INTELLIGENCE_DB, data), + membershipStatements: data => scimPrincipalStatements(env.INTELLIGENCE_DB, data), + teamStatements: data => scimTeamStatements(env.INTELLIGENCE_DB, data), + }); + if (url.pathname === "/v1/managed-state/credits") return handleCreditsRequest(request, env); + if (url.pathname === "/v1/managed-state/payments") return handlePaymentStoreRequest(request, env); + if (url.pathname === "/v1/managed-state/idempotency") return handleIdempotencyRequest(request, env, { tenantContext }); + if (url.pathname === "/v1/managed-state/responses") return handleResponsesStateRequest(request, env, { tenantContext }); + if (url.pathname === "/v1/managed-state/context-pages") return handleContextPageStateRequest(request, env, { tenantContext }); if (url.pathname === "/v1/managed-state/usage-receipts") return handleUsageReceiptStoreRequest(request, env); if (url.pathname === "/v1/managed-state/reservations" && reservations) return reservations(request, env); if (url.pathname === "/v1/managed-state/tool-grants" && toolGrants) return toolGrants(request, env); diff --git a/worker/reservations-d1.mjs b/worker/reservations-d1.mjs index 1cbaa868..286b0a5c 100644 --- a/worker/reservations-d1.mjs +++ b/worker/reservations-d1.mjs @@ -1,3 +1,7 @@ +import { createGovernanceLifecycle, governanceEnabled, handleTenantGovernanceRequest, GovernanceError, intersectGrants } from "./tenant-governance-d1.mjs"; +import { createCreditsLifecycle, CreditsAdmissionError, integerMicroUsd, nativeTariffRevision, reconcileCreditsReservation } from "./credits-admission.mjs"; +import { creditsEnforcement } from "./credits-d1.mjs"; +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; import { recordSettledUsage } from "./usage-receipts.mjs"; import { retentionRequestId } from "./retention-policy.mjs"; /** Private, atomic monetary holds and injected request lifecycle. */ @@ -245,21 +249,23 @@ export async function handleReservationsRequest(request, env, { now = Date.now } .bind(body.id).all(); result = { transitions: rows.results }; } else result = { reservation: publicRow(await db.prepare("SELECT * FROM usage_reservations WHERE id=?").bind(body.id).first()) }; - if (body.operation === "transition" && body.state === "reconciled" && result.applied) { + if (body.operation === "transition" && body.state === "reconciled") { const row = result.reservation; + await reconcileCreditsReservation(env, row, body.transition_id); + if (result.applied) await recordSettledUsage(env, row.principal, body.transition_id, { reservation_id: row.id, settlement_id: row.settlement_id, state: row.state, cost_usd: row.cost_usd, cost_basis: row.basis, input_tokens: row.input_tokens, output_tokens: row.output_tokens }); } return reply(result); } catch (error) { - const failure = error instanceof ReservationError ? error : new ReservationError(); + const failure = error instanceof ReservationError || error instanceof CreditsAdmissionError ? error : new ReservationError(); return reply({ error: { code: failure.code, message: "The usage reservation operation could not be completed." } }, failure.status); } } /** One instance per native request. Identity and pricing come from verified admission. */ -export function createReservationLifecycle(env, { call, identity }) { - let reservation, finalEvent; +export function createReservationLifecycle(env, { call, identity, governance, credits, workspace = false, metadata = async event => event }) { + let reservation, finalEvent, linkedIdentity; let dispatching; const transition = async (state, fields = {}) => { const result = await call({ version: 1, operation: "transition", id: reservation.id, revision: reservation.revision, @@ -273,7 +279,7 @@ export function createReservationLifecycle(env, { call, identity }) { if (!reservationSettings(env).enabled) return; const values = await identity(context); if (values.daily_budget_usd == null && values.monthly_budget_usd == null) return; - const result = await call({ version: 1, operation: "reserve", id: crypto.randomUUID().replaceAll("-", ""), ...values }); + const result = await call({ version: 1, operation: "reserve", id: linkedIdentity ?? crypto.randomUUID().replaceAll("-", ""), ...values }); if (result.error) throw new ReservationError(result.error.code, result.error.code === "budget_exceeded" ? 429 : 503); reservation = result.reservation; if (finalEvent) await hooks.finalize(finalEvent); @@ -299,7 +305,44 @@ export function createReservationLifecycle(env, { call, identity }) { output_tokens: context.cancellationOutcome?.ambiguous ? null : row.output_tokens ?? null }); }, }; - return hooks; + if (!governance && !credits) return hooks; + let admitted, pending, eventMetadata; + const describe = async event => eventMetadata ??= await metadata(event); + const finalize = async event => { + finalEvent = event; + if (pending) await pending.catch(() => {}); + const results = await Promise.allSettled([governance?.finalize(event), hooks.finalize(event), credits?.finalize(event)]); + const failure = results.find(result => result.status === "rejected"); + if (failure) throw failure.reason; + }; + return { + async authorize(event) { await governance?.authorize(await describe(event)); }, + async admit(event) { + if (admitted) return; + if (pending) return pending; + pending = (async () => { + const values = await identity(event, Boolean(workspace || await credits?.requiresPricing())); + linkedIdentity ??= crypto.randomUUID().replaceAll("-", ""); + const priced = { ...await describe(event), scoped_id: workspace ? `tg_${linkedIdentity}` : linkedIdentity, amount: values.estimate_usd == null ? null : integerMicroUsd(values.estimate_usd), + key_daily: values.daily_budget_usd == null ? null : integerMicroUsd(values.daily_budget_usd), + key_monthly: values.monthly_budget_usd == null ? null : integerMicroUsd(values.monthly_budget_usd), + base_day: integerMicroUsd(values.day_spent_usd ?? 0), base_month: integerMicroUsd(values.month_spent_usd ?? 0), + tariff_revision: nativeTariffRevision(env) }; + await governance?.admit(priced); + if (!workspace) await hooks.admit(event); + await credits?.admit(priced); + admitted = true; + })(); + try { await pending; } + catch (error) { pending = null; await finalize({ ...event, handedOff: false }); throw error; } + pending = null; + if (finalEvent) await finalize(finalEvent); + }, + async before_dispatch(event) { + await governance?.before_dispatch(event); await hooks.before_dispatch(event); await credits?.before_dispatch(event); + }, + finalize, + }; } function nativePrice(table, provider, model, input, output, units) { @@ -335,7 +378,7 @@ async function boundedReservationBody(request) { } finally { void reader.cancel().catch(() => {}); } } -async function nativeReservationIdentity(request, env, authority, context) { +async function nativeReservationIdentity(request, env, authority, context, forcePrice = false) { let principal = authority.principal; if (!principal?.id) throw new ReservationError(); // Native bootstrap authentication uses the environment; stored controls are optional metadata. @@ -346,8 +389,9 @@ async function nativeReservationIdentity(request, env, authority, context) { principal = { ...principal, ...stored }; } const limits = { daily_budget_usd: principal.daily_budget_usd ?? null, monthly_budget_usd: principal.monthly_budget_usd ?? null }; - if (limits.daily_budget_usd === null && limits.monthly_budget_usd === null) return limits; - const identity = `edge:${await retentionRequestId(`native-admin:${principal.id}`)}`; + if (!forcePrice && limits.daily_budget_usd === null && limits.monthly_budget_usd === null) return limits; + const legacyIdentity = `edge:${await retentionRequestId(`native-admin:${principal.id}`)}`; + const identity = organisationsEnabled(env) ? tenantStorageKey(legacyIdentity, authority.tenantContext) : legacyIdentity; let estimate = 0; if (!context.cacheServed) { const { body, size } = await boundedReservationBody(request); @@ -363,6 +407,8 @@ async function nativeReservationIdentity(request, env, authority, context) { if (!costs.length || costs.some(cost => cost === null)) throw new ReservationError("unpriced_reservation"); estimate = Math.max(...costs); } + if (limits.daily_budget_usd === null && limits.monthly_budget_usd === null) + return { principal: identity, estimate_usd: estimate, ...limits, day_spent_usd: 0, month_spent_usd: 0 }; const day = new Date().toISOString().slice(0, 10); const baseline = await env.INTELLIGENCE_DB.prepare(`SELECT COALESCE(SUM(CASE WHEN day = ? THEN cost_usd ELSE 0 END), 0) AS day_spent_usd, COALESCE(SUM(cost_usd), 0) AS month_spent_usd FROM usage_daily WHERE principal = ? AND day >= ? AND day <= ?`) @@ -383,11 +429,46 @@ async function boundedReservationOperation(action) { } /** Only server-verified principal metadata can impose native monetary limits. */ -export function nativeReservationLifecycle(request, env, authority) { +export function nativeReservationLifecycle(request, env, authority, { identity: suppliedIdentity } = {}) { + const workspace = governanceEnabled(env) && authority.tenantContext?.org_id != null; + const governance = governanceEnabled(env) ? createGovernanceLifecycle(env, { + tenant_resolver: () => authority.tenantContext, + call: async body => { + const response = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ...body, rpc: true }), + }), env); + return { ...await response.json(), status: response.status }; + }, + }) : null; + const credits = createCreditsLifecycle(env, { context: authority.tenantContext }); + let values; + const identity = async (event, force = false) => { + if (values && (!force || values.estimate_usd != null)) return values; + try { values = suppliedIdentity ? await suppliedIdentity(event, force) + : await boundedReservationOperation(() => nativeReservationIdentity(request, env, authority, event, force)); } + catch (error) { if (await credits.requiresPricing() && error.code === "unpriced_reservation") throw new CreditsAdmissionError("credits_unpriced"); throw error; } + return values; + }; + const metadata = async event => { + const model = event.model?.includes(":") ? event.model : `${authority.provider}:${event.model}`; + if (!workspace) return { ...event, model, key_allowed: true, tools: [], key_tools: [] }; + let principal = authority.principal; + if (governanceEnabled(env) && env.INTELLIGENCE_DB) { + const stored = await env.INTELLIGENCE_DB.prepare(`SELECT allowed_models FROM control_users WHERE username=? AND revoked_at IS NULL`) + .bind(principal.id).first(); + principal = { ...principal, ...stored }; + } + const patterns = principal.allowed_models == null ? null : String(principal.allowed_models).split(","); + const key_allowed = patterns === null || intersectGrants(true, model, [{ models: patterns }]); + if (governanceEnabled(env) && !key_allowed) throw new GovernanceError("model_not_allowed", 403); + const { body } = await boundedReservationBody(request); + const tools = (Array.isArray(body.tools) ? body.tools : []).map(tool => tool?.function?.name ?? tool?.name).filter(name => typeof name === "string"); + return { ...event, model, key_allowed, tools, key_tools: authority.key_tools ?? tools }; + }; return createReservationLifecycle(env, { call: body => boundedReservationOperation(async () => { const response = await handleReservationsRequest(new Request("http://intelligence.internal/v1/reservations", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), }), env); return response.json(); - }), identity: context => boundedReservationOperation(() => nativeReservationIdentity(request, env, authority, context)) }); + }), identity, ...(governance || creditsEnforcement(env) !== "off" ? { governance, credits, workspace, metadata } : {}) }); } diff --git a/worker/responses-state-d1.mjs b/worker/responses-state-d1.mjs index c4617a51..13583ab2 100644 --- a/worker/responses-state-d1.mjs +++ b/worker/responses-state-d1.mjs @@ -1,3 +1,4 @@ +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; /** Principal-owned metadata and bounded immutable continuation bodies. */ import { boundedBody } from "./control-users-d1.mjs"; @@ -133,7 +134,7 @@ async function put(db, bucket, body, now) { return reply("responses_store_unavailable", 503); } -export async function handleResponsesStateRequest(request, env) { +export async function handleResponsesStateRequest(request, env, { tenantContext } = {}) { const url = new URL(request.url); if (request.method !== "POST" || url.origin !== "http://intelligence.internal" || url.pathname !== PATH || url.search || url.hash || url.username || url.password) return reply("not_found", 404); @@ -144,6 +145,7 @@ export async function handleResponsesStateRequest(request, env) { body = JSON.parse(await boundedBody(request, MAX_DOCUMENT_BYTES)); if (!validBody(body)) throw Error(); } catch { return reply("invalid_responses_state_request", 400); } + if (organisationsEnabled(env) && tenantContext && body.owner) body.owner = tenantStorageKey(body.owner, tenantContext); const db = env.INTELLIGENCE_DB, bucket = env.multillm_media; if (!db || !bucket) return reply("responses_store_unavailable", 503); const now = Math.floor(Date.now() / 1000); diff --git a/worker/scim-d1.mjs b/worker/scim-d1.mjs index b1379d7f..9443f3ae 100644 --- a/worker/scim-d1.mjs +++ b/worker/scim-d1.mjs @@ -1,5 +1,5 @@ /** Atomic SCIM identity, account, team and security-revision storage. */ -import { boundedBody, USER_FIELDS, validUser } from "./control-users-d1.mjs"; +import { boundedBody, USER_FIELDS, validUser, scimAccountStatements } from "./control-users-d1.mjs"; import { commitRevision, requireRevisionSchema, revisionSyncSettings } from "./config-revision.mjs"; const PATH = "/v1/managed-state/scim"; @@ -124,16 +124,6 @@ function resourceStatements(db, body) { return [statement, casGuard(db, body)]; } -function accountStatements(db, body, priorAccount) { - if (!body.account) return []; - const { account, expected } = body; - if (expected === 0) return [db.prepare(`INSERT INTO control_users (${USER_FIELDS.join(",")}) VALUES (${USER_FIELDS.map(() => "?").join(",")})`) - .bind(...USER_FIELDS.map(key => account[key]))]; - return [db.prepare(`UPDATE control_users SET api_key_hash=?,api_key_prefix=?,revoked_at=? - WHERE username=? AND is_admin=0 AND api_key_hash=? AND api_key_prefix=? AND revoked_at IS ?`) - .bind(account.api_key_hash, account.api_key_prefix, account.revoked_at, account.username, - priorAccount.api_key_hash, priorAccount.api_key_prefix, priorAccount.revoked_at), casGuard(db, body)]; -} async function prepareAccount(db, body, old) { if (body.kind !== "Users") return null; @@ -180,9 +170,9 @@ async function put(db, body, collaborators) { } const priorAccount = await prepareAccount(db, body, old); const statements = resourceStatements(db, body); - statements.push(...accountStatements(db, body, priorAccount)); - if (kind === "Users" && expected === 0) { - statements.push(...collaboratorStatements(collaborators.principalStatements, { org_id: org, resource: row })); + statements.push(...scimAccountStatements(db, body, priorAccount, () => casGuard(db, body))); + if (kind === "Users" && (expected === 0 || collaborators.membershipStatements)) { + statements.push(...collaboratorStatements(expected === 0 ? collaborators.principalStatements : collaborators.membershipStatements, { org_id: org, resource: row })); } if (kind === "Groups") { statements.push(...collaboratorStatements(collaborators.teamStatements, { org_id: org, resource: row, prior: old, deactivated: body.deactivate })); diff --git a/worker/semantic-generation-cache.mjs b/worker/semantic-generation-cache.mjs index fb9e2796..2cded070 100644 --- a/worker/semantic-generation-cache.mjs +++ b/worker/semantic-generation-cache.mjs @@ -1,9 +1,13 @@ +import { governanceEnabled, GovernanceError } from "./tenant-governance-d1.mjs"; +import { creditsEnforcement } from "./credits-d1.mjs"; +import { CreditsAdmissionError } from "./credits-admission.mjs"; +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; /** Native semantic lookup after authentication and policy, before generation dispatch. */ import { digest, MAX_BODY_BYTES, completeCacheBody } from "./generation-cache-d1.mjs"; import { SemanticCacheD1, validVector, vectorSimilarity, schemaMissing, schemaErrorResponse } from "./semantic-cache-d1.mjs"; import { retentionAllowsContent } from "./retention-policy.mjs"; import { cacheServedEvent } from "./exact-generation-cache.mjs"; -import { createReservationLifecycle, handleReservationsRequest, reservationSettings } from "./reservations-d1.mjs"; +import { createReservationLifecycle, nativeReservationLifecycle, handleReservationsRequest, reservationSettings } from "./reservations-d1.mjs"; import { recordNativeUsage } from "./usage-ledger-d1.mjs"; const object = value => value && typeof value === "object" && !Array.isArray(value); const MODEL = /^[a-z][a-z0-9-]*:[A-Za-z0-9@][A-Za-z0-9._:/+@-]{0,255}$/; @@ -63,6 +67,7 @@ export async function semanticPartition(principal, provider, route, payload, con retention_config: env.CONTENT_RETENTION_POLICY_JSON ?? "", retention_enabled: env.CONTENT_RETENTION_ENABLED ?? "", secret_policy: env.SECRET_SCAN_DEFAULT ?? "redact", secret_enabled: env.SECRET_SCAN_ENABLED ?? "", security: context.semanticSecurity ?? {}}; + if (organisationsEnabled(env)) principal = tenantStorageKey(principal, context.tenantContext); return {principal_hash: await digest(principal), partition_hash: await digest(JSON.stringify(canonical(invariant))), model_revision: await digest(JSON.stringify([policy.embedding_model, policy.model_revision ?? "1"]))}; } @@ -174,7 +179,10 @@ export async function prepareSemanticCache(request, env, authority, context, col } let vector; try {vector = await paidEmbedding(env, authority, request, policy, payload.messages.at(-1).content, price, collaborators);} - catch {warn("embedding_unavailable"); return null;} + catch (error) { + if (error instanceof CreditsAdmissionError || error instanceof GovernanceError) throw error; + warn("embedding_unavailable"); return null; + } if (!vector || request.signal.aborted) return null; const unchanged = async () => semanticCacheSettings(env).enabled && retentionAllowsContent(context.retentionPolicy) && JSON.stringify(await current()) === JSON.stringify(identity) && !request.signal.aborted @@ -235,6 +243,25 @@ export function createNativeSemanticCollaborators(request, env, ctx, authority, const limits = { daily_budget_usd: account.daily_budget_usd ?? null, monthly_budget_usd: account.monthly_budget_usd ?? null }; const budgeted = Object.values(limits).some(value => value !== null); if (budgeted && !reservationSettings(env).enabled) return false; + if (governanceEnabled(env) || creditsEnforcement(env) !== "off") { + const embeddingRequest = new Request(request.url, { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: model.slice(model.indexOf(":") + 1), input: "" }) }); + const legacyPrincipal = await ledgerPrincipal(); + const principal = organisationsEnabled(env) ? tenantStorageKey(legacyPrincipal, authority.tenantContext) : legacyPrincipal; + const day = new Date().toISOString().slice(0, 10); + const baseline = budgeted ? await env.INTELLIGENCE_DB.prepare(`SELECT + COALESCE(SUM(CASE WHEN day=? THEN cost_usd ELSE 0 END),0) AS day_spent_usd, + COALESCE(SUM(cost_usd),0) AS month_spent_usd FROM usage_daily WHERE principal=? AND day>=? AND day<=?`) + .bind(day, principal, `${day.slice(0,7)}-01`, day).first() : { day_spent_usd: 0, month_spent_usd: 0 }; + if (!baseline) throw Error("semantic_embedding_budget_unavailable"); + const lifecycle = nativeReservationLifecycle(embeddingRequest, env, { ...authority, principal: account }, { + identity: async () => ({ principal, estimate_usd: price, ...limits, ...baseline }), + }); + const event = { ...context, model: model.slice(model.indexOf(":")+1), provider: authority.provider }; + await lifecycle.authorize(event); await lifecycle.admit(event); + let handedOff = false; + return { ...lifecycle, get handedOff() { return handedOff; }, handoff() { handedOff = true; } }; + } const lifecycle = createReservationLifecycle(env, { call: async body => { const response = await handleReservationsRequest(new Request("http://intelligence.internal/v1/reservations", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), @@ -264,6 +291,7 @@ export function createNativeSemanticCollaborators(request, env, ctx, authority, headers.set("content-type", "application/json"); headers.set("cache-control", "no-store"); const embeddingAuthority = { ...authority, provider: model.split(":", 1)[0], route: authority.route.replace(/\/chat\/completions$/, "/embeddings") }; + reservation?.handoff?.(); const response = await fetcher(new Request(target, { method: "POST", headers, signal, redirect: "manual", body: JSON.stringify({ model: model.slice(model.indexOf(":") + 1), input: text }) }), env, embeddingAuthority); const bytes = await boundedBytes(response.body, MAX_BODY_BYTES); @@ -271,10 +299,10 @@ export function createNativeSemanticCollaborators(request, env, ctx, authority, return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); }, async accountEmbedding(event) { - const handedOff = event.submission_outcome !== "before-dispatch"; + const handedOff = typeof event.reservation?.handedOff === "boolean" ? event.reservation.handedOff : event.submission_outcome !== "before-dispatch"; const outcome = event.status === 200 ? "success" : event.status === 499 ? "canceled" : "transport_error"; try { - await recordNativeUsage(env, { ...event, principal: await ledgerPrincipal(), requestId: crypto.randomUUID(), + await recordNativeUsage(env, { ...event, ...(authority.tenantContext ? { tenantContext: authority.tenantContext } : {}), principal: await ledgerPrincipal(), requestId: crypto.randomUUID(), model: event.model.slice(event.model.indexOf(":") + 1), outcome, ttft_ms: null, usage_basis: event.cost_basis === "usage" ? "measured" : "estimated" }, ctx); } finally { diff --git a/worker/tenant-governance-d1.mjs b/worker/tenant-governance-d1.mjs index aef3f34f..8e190d02 100644 --- a/worker/tenant-governance-d1.mjs +++ b/worker/tenant-governance-d1.mjs @@ -1,3 +1,6 @@ +import { createHash } from "node:crypto"; +import { contextOwner } from "./credits-d1.mjs"; +import { integerMicroUsd, reconcileCreditsReservation } from "./credits-admission.mjs"; /** Private governance domain: intersected grants, CAS policy and atomic quota components. */ import { TenantContext, AuthorityOperation, legacyTenant, MAX_INTEGER } from "./enterprise-contract.mjs"; @@ -7,6 +10,7 @@ const FIELDS = { get: ["org_id", "team_id"], put: ["org_id", "team_id", "actor", "revision", "policy"], policies: ["context"], usage: ["context", "role", "since", "until"], reserve: ["id", "context", "amount", "day", "month", "key_daily", "key_monthly", "base_day", "base_month"], + reconcile: ["id", "revision", "cost", "transition_id", "admin", "reason", "evidence", "authorized_adjustment"], dispatch: ["id"], settle: ["id", "cost", "provider", "model", "price_basis", "before_dispatch"], }; const SCHEMA = [ @@ -78,7 +82,16 @@ function validate(body) { if (op === "usage") for (const key of ["since", "until"]) { if (body[key] !== undefined && (typeof body[key] !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(body[key]))) fail(); } - if (["reserve", "dispatch", "settle"].includes(op)) opaque(body.id); + if (["reserve", "dispatch", "settle", "reconcile"].includes(op)) opaque(body.id); + if (op === "reconcile") { + if (body.authorized_adjustment !== undefined && typeof body.authorized_adjustment !== "boolean") fail(); + if (!integer(body.revision) || typeof body.transition_id !== "string" || !/^[a-f0-9]{32}$/.test(body.transition_id) + || body.cost !== null && !integer(body.cost)) fail(); + if (body.admin !== true) fail("admin_required", 403); + const label = value => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value) && !/[\r\n]/.test(value); + if (!label(body.reason) || (body.authorized_adjustment === true ? body.evidence != null : !label(body.evidence))) + fail("reconciliation_evidence_required"); + } if (op === "reserve") { for (const key of ["amount", "base_day", "base_month"]) if (!integer(body[key])) fail(); for (const key of ["key_daily", "key_monthly"]) if (body[key] !== null && !integer(body[key])) fail(); @@ -196,6 +209,36 @@ async function settle(db, body) { if (!results[0].results.length) fail("reservation_conflict", 409); return { state }; } +async function reconcile(db, body, now) { + const adjustment = body.authorized_adjustment ?? false; + const fingerprint = "reconcile:" + createHash("sha256").update(JSON.stringify([ + body.id, body.revision, body.cost, body.reason, body.evidence ?? null, adjustment])).digest("hex"); + const row = await db.prepare("SELECT * FROM tenant_governance_reservations WHERE id=?").bind(body.id).first(); + if (!row) fail("reservation_conflict", 409); + const readAudit = () => db.prepare("SELECT actor,kind FROM tenant_governance_audit WHERE operation_id=?").bind(body.transition_id).first(); + const previous = await readAudit(); + let applied = false; + if (!previous) { + if (row.revision !== body.revision || !["dispatched", "unknown"].includes(row.state)) fail("reservation_conflict", 409); + const results = await batch(db, [ + db.prepare(`UPDATE tenant_governance_reservations SET state=?,charged=?,revision=revision+1,operation_id=? + WHERE id=? AND revision=? AND state IN ('dispatched','unknown') + AND NOT EXISTS (SELECT 1 FROM tenant_governance_audit WHERE operation_id=?) RETURNING id`) + .bind(body.cost === null ? "unknown" : "settled", body.cost, body.transition_id, body.id, body.revision, body.transition_id), + db.prepare(`INSERT OR IGNORE INTO tenant_governance_audit + SELECT ?,id,org_id,team_id,revision,?,? FROM tenant_governance_reservations WHERE id=? AND operation_id=?`) + .bind(body.transition_id, fingerprint, new Date(now).toISOString(), body.id, body.transition_id), + ]); + applied = results[0].results.length === 1; + } + const audit = previous ?? await readAudit(); + if (audit?.actor !== body.id || audit?.kind !== fingerprint) fail("reservation_conflict", 409); + const principal = contextOwner(new TenantContext({ principal_id: row.principal_id, org_id: row.org_id, team_id: row.team_id || null })); + return { applied, reservation: { id: body.id, principal, day: row.day, month: row.month, revision: body.revision + 1, + state: body.cost === null ? "unknown" : "reconciled", cost_usd: body.cost === null ? null : body.cost / 1_000_000, + transition_id: body.transition_id, settlement_id: body.id, basis: adjustment ? "adjustment" : "provider", + input_tokens: null, output_tokens: null } }; +} async function usage(db, body) { const c = body.context, all = permitted(body.role, "all_usage"); const where = "org_id=? AND (?='' OR team_id=?) AND (?=1 OR principal_id=?) AND day>=? AND day<=?"; @@ -256,6 +299,10 @@ export async function handleTenantGovernanceRequest(request, env, { now = Date.n if (["get", "put"].includes(body.operation)) result = await admin(db, body, now()); else if (body.operation === "policies") result = await policies(db, body.context); else if (body.operation === "reserve") result = await reserve(db, body, now()); + else if (body.operation === "reconcile") { + result = await reconcile(db, body, now()); + if (result.reservation.cost_usd !== null) await reconcileCreditsReservation(env, result.reservation, body.transition_id); + } else if (body.operation === "usage") result = await usage(db, body); else result = await settle(db, body); return reply(result); @@ -275,7 +322,7 @@ function defaultTenantResolver(event) { } /** Named native hooks consume trusted admission metadata; no caller header selects tenancy. */ export function createGovernanceLifecycle(env, { call, tenant_resolver = defaultTenantResolver }) { - let id, finished = false; + let id, finished = false, authorised = false; const rpc = async (operation, values) => { const result = await call({ version: 1, operation, ...values }); const failure = result.error ?? (result.decision?.allowed === false ? result.decision : null); @@ -283,16 +330,25 @@ export function createGovernanceLifecycle(env, { call, tenant_resolver = default { ...(failure.level ? { level: failure.level } : {}), ...(failure.period ? { period: failure.period } : {}) }); return result; }; + const authorize = async event => { + if (!governanceEnabled(env) || authorised) return; + const context = await tenant_resolver(event); + if (!(context instanceof TenantContext)) fail("tenant_scope_denied", 403); + if (context.org_id === null) return; + const rows = (await rpc("policies", { context })).policies; + if (!intersectGrants(event.key_allowed === true, event.model, rows)) fail("model_not_allowed", 403); + for (const tool of event.tools ?? []) if (!intersectGrants(event.key_tools?.includes(tool) === true, tool, rows, "tools")) fail("tool_not_allowed", 403); + authorised = true; + }; return { + authorize, async admit(event) { - if (!governanceEnabled(env)) return; + if (!governanceEnabled(env) || id) return; const context = await tenant_resolver(event); if (!(context instanceof TenantContext)) fail("tenant_scope_denied", 403); if (context.org_id === null) return; - const rows = (await rpc("policies", { context })).policies; - if (!intersectGrants(event.key_allowed === true, event.model, rows)) fail("model_not_allowed", 403); - for (const tool of event.tools ?? []) if (!intersectGrants(event.key_tools?.includes(tool) === true, tool, rows, "tools")) fail("tool_not_allowed", 403); - const candidateId = `tg_${crypto.randomUUID().replaceAll("-", "")}`; + await authorize(event); + const candidateId = event.scoped_id ?? `tg_${crypto.randomUUID().replaceAll("-", "")}`; const day = new Date().toISOString().slice(0, 10); await rpc("reserve", { id: candidateId, context, amount: event.amount, day, month: day.slice(0, 7), key_daily: event.key_daily ?? null, key_monthly: event.key_monthly ?? null, @@ -302,11 +358,11 @@ export function createGovernanceLifecycle(env, { call, tenant_resolver = default async before_dispatch() { if (id && !finished) await rpc("dispatch", { id }); }, async finalize(event) { if (!id || finished) return; - const row = event.usage ?? event, model = row.selected_model ?? null; + const row = event.usage ?? event, model = row.selected_model ?? (row.model ? `${row.provider}:${row.model}` : null); const cached = row.cost_basis === "cache"; const released = event.handedOff === false; const measured = !event.cancellationOutcome?.ambiguous && row.cost_basis === "usage"; - await rpc("settle", { id, cost: released || cached ? 0 : measured ? row.cost_micro_usd ?? null : null, + await rpc("settle", { id, cost: released || cached ? 0 : measured ? row.cost_micro_usd ?? (row.cost_usd == null ? null : integerMicroUsd(row.cost_usd)) : null, provider: model?.includes(":") ? model.split(":", 1)[0] : null, model, price_basis: cached ? "cache" : released ? "released" : row.cost_basis ?? null, before_dispatch: released && !cached }); finished = true; diff --git a/worker/tenants-d1.mjs b/worker/tenants-d1.mjs index 6c310fd7..c0454334 100644 --- a/worker/tenants-d1.mjs +++ b/worker/tenants-d1.mjs @@ -42,6 +42,13 @@ function legacy(principal) { export function tenantNamespace(context) { return context.org_id === null ? "" : `org:${context.org_id}${context.team_id === null ? "" : `/team:${context.team_id}`}`; } +/** Trusted callers append workspace scope; legacy bytes remain untouched. */ +export function tenantStorageKey(value, context) { + if (context == null) return value; + if (!(context instanceof TenantContext)) throw new TenantError("workspace_forbidden", 403); + const namespace = tenantNamespace(context); + return namespace ? createHash("sha256").update(JSON.stringify([value, namespace])).digest("hex") : value; +} const one = (db, sql, values = []) => db.prepare(sql).bind(...values).first(); const rows = async (db, sql, values = []) => (await db.prepare(sql).bind(...values).all()).results; async function requireSchema(db) { @@ -260,3 +267,46 @@ export async function handleTenantRequest(request, env) { return (error instanceof TenantError ? error : new TenantError("tenant_storage_unavailable", 503)).response(true); } } + +function scimTenantGuard(db, org, principal) { + // A failed conditional mutation aborts the entire SCIM/account/revision batch. + return db.prepare(`INSERT INTO tenant_memberships (org_id,principal,role,status,revision) + SELECT ?,?,'member','active',0 WHERE changes()!=1`).bind(org, principal); +} +/** No binding or privileged role is created by provisioning. */ +export function scimPrincipalStatements(db, { org_id, resource }) { + identifier(org_id); requireText(resource.userName); + return [db.prepare(`INSERT INTO tenant_memberships (org_id,principal,team_id,role,status,revision) + SELECT ?1,?2,NULL,'member',?3,1 WHERE EXISTS + (SELECT 1 FROM tenant_organisations WHERE id=?1 AND status='active') + AND ((SELECT COUNT(*) FROM tenant_memberships WHERE org_id=?1)<1000 + OR EXISTS (SELECT 1 FROM tenant_memberships WHERE org_id=?1 AND principal=?2)) + ON CONFLICT(org_id,principal) DO UPDATE SET status=excluded.status,revision=tenant_memberships.revision+1 + WHERE tenant_memberships.role='member'`) + .bind(org_id, resource.userName, resource.active ? "active" : "deactivated"), + scimTenantGuard(db, org_id, resource.userName)]; +} +/** Team and membership mutations join the SCIM resource transaction. */ +export function scimTeamStatements(db, { org_id, resource, prior, deactivated }) { + identifier(org_id); identifier(resource.id); requireText(resource.displayName); + const statements = [db.prepare(`INSERT INTO tenant_teams (id,org_id,name,status,revision) + SELECT ?1,?2,?3,?4,1 WHERE EXISTS + (SELECT 1 FROM tenant_organisations WHERE id=?2 AND status='active') + AND ((SELECT COUNT(*) FROM tenant_teams WHERE org_id=?2)<100 + OR EXISTS (SELECT 1 FROM tenant_teams WHERE id=?1 AND org_id=?2)) + ON CONFLICT(id) DO UPDATE SET name=excluded.name,status=excluded.status,revision=tenant_teams.revision+1 + WHERE tenant_teams.org_id=excluded.org_id`) + .bind(resource.id, org_id, resource.displayName, deactivated ? "deactivated" : "active"), + scimTenantGuard(db, org_id, resource.id)]; + // Explicit group updates may move members of this group, never members of another team. + statements.push(db.prepare(`UPDATE tenant_memberships SET team_id=NULL,revision=revision+1 + WHERE org_id=? AND team_id=? AND role='member'`).bind(org_id, resource.id)); + for (const member of deactivated ? [] : resource.members) { + statements.push(db.prepare(`UPDATE tenant_memberships SET team_id=?1,revision=revision+1 + WHERE org_id=?2 AND principal=(SELECT user_name FROM scim_resources + WHERE org_id=?2 AND kind='Users' AND id=?3 AND deactivated=0) + AND role='member' AND status='active' AND team_id IS NULL`) + .bind(resource.id, org_id, member.value), scimTenantGuard(db, org_id, member.value)); + } + return statements; +} diff --git a/worker/usage-ledger-d1.mjs b/worker/usage-ledger-d1.mjs index cf3e0b1a..bb3692d6 100644 --- a/worker/usage-ledger-d1.mjs +++ b/worker/usage-ledger-d1.mjs @@ -1,3 +1,4 @@ +import { organisationsEnabled, tenantStorageKey } from "./tenants-d1.mjs"; /** * Durable usage ledger in D1, reachable only through the Container's private outbound * handler. The Container batches billable requests and flushes them here; every flush @@ -199,7 +200,7 @@ export async function recordNativeUsage(env, event, ctx) { const model = event.model ? `${event.provider}:${event.model}` : null; const kind = event.endpoint.endsWith("/responses") ? "responses" : event.endpoint.endsWith("/embeddings") ? "embeddings" : event.endpoint.endsWith("/images/generations") ? "images" : "chat"; - const row = { at: new Date().toISOString(), principal: event.principal, key_prefix: null, + const row = { at: new Date().toISOString(), principal: organisationsEnabled(env) ? tenantStorageKey(event.principal, event.tenantContext) : event.principal, key_prefix: null, kind, endpoint: event.endpoint, requested_model: model, selected_model: model, status: event.status, latency_ms: event.duration_ms, input_tokens: event.input_tokens, output_tokens: event.output_tokens, cost_usd: event.cost_usd, cost_basis: event.cost_basis, From 45109924f44b761c6e9d6893de5285a006ec03a9 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 15:11:02 +0530 Subject: [PATCH 9/9] fix(ci): run the credits parity check with the CI interpreter --- tests/test_credits_ledger_worker.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_credits_ledger_worker.mjs b/tests/test_credits_ledger_worker.mjs index b4ca9eb9..61e2d488 100644 --- a/tests/test_credits_ledger_worker.mjs +++ b/tests/test_credits_ledger_worker.mjs @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { DatabaseSync } from 'node:sqlite'; import { spawnSync } from 'node:child_process'; import { handleCreditsRequest, appendCredit, appendPaymentCredit, readCredits, registerCreditAuthority, creditsEnabled, creditsEnforcement, contextOwner } from '../worker/credits-d1.mjs'; import { AuthorityOperation, TenantContext, callAuthority, registerEnterpriseAdapters } from '../worker/enterprise-contract.mjs'; +const PYTHON = process.env.PYTHON || (existsSync('.venv/bin/python') ? '.venv/bin/python' : 'python3'); + const migration = readFileSync(new URL('../intelligence-migrations/0037_credits_ledger.sql', import.meta.url), 'utf8'); const vectors = JSON.parse(readFileSync(new URL('../docs/credits-ledger.md', import.meta.url), 'utf8').split('```json\n')[1].split('```')[0]); @@ -220,7 +222,7 @@ test('Python and D1 return identical ledger vectors', async t => { await (vector.operation === 'payment_append' ? appendPaymentCredit : appendCredit)(db, vector.request); } const code = `import os,sys,json,tempfile\nfrom pathlib import Path\nsys.path.insert(0,os.getcwd())\nfrom services.credits_ledger import SqlCreditsLedger\ntext=Path('docs/credits-ledger.md').read_text()\nvectors=json.loads(text.split('\x60\x60\x60json\\n',1)[1].split('\x60\x60\x60',1)[0])\nwith tempfile.TemporaryDirectory() as d:\n store=SqlCreditsLedger(Path(d)/'credits.sqlite3',initialize=True)\n for v in vectors:\n if 'request' in v and not v.get('error'): (store.append_payment if v.get('operation') == 'payment_append' else store.append)(**v['request'])\n print(json.dumps(store.read('alice')))\n`; - const python = spawnSync('/home/mysterious/storage/github/MultiLLM-Proxy/.venv/bin/python', ['-I', '-c', code], { + const python = spawnSync(PYTHON, ['-I', '-c', code], { cwd: new URL('..', import.meta.url), encoding: 'utf8', timeout: 10000, }); assert.equal(python.status, 0, python.stderr);