From 649f8f9c136662981ba87e416bbd6362adc118a1 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 12:35:56 +0530 Subject: [PATCH] feat(governance): add opt-in hierarchical grants, budgets and attribution TENANT_GOVERNANCE_ENABLED (off by default) applies organisation and team policies to requests in a verified workspace. Model and tool grants intersect with the key's own grants and never widen them. Budgets at every configured level, plus the key's own limits, are reserved together in one atomic operation and settled together; an over-limit level denies the request before dispatch, and an unknown cost keeps its hold at every level. Roles come from a static permission table. Administrators edit policies through /admin/organisations/[/teams/]/governance with If-Match and audit rows, and /v1/usage gains a workspace breakdown for callers in a workspace. Requests outside a workspace are unchanged (migration 0034). --- docs/tenant-governance.md | 76 +++ .../0034_tenant_governance.sql | 30 ++ package.json | 2 +- routes/usage.py | 77 ++- services/budget_service.py | 18 +- services/intelligence_policy.py | 4 +- services/key_controls.py | 23 +- services/tenant_governance.py | 485 ++++++++++++++++++ tests/fixtures/secret_firewall_coverage.json | 3 + tests/test_tenant_governance.py | 364 +++++++++++++ tests/test_tenant_governance_worker.mjs | 214 ++++++++ worker/container-env.mjs | 1 + worker/d1-schema.mjs | 2 +- worker/tenant-governance-d1.mjs | 315 ++++++++++++ 14 files changed, 1601 insertions(+), 13 deletions(-) create mode 100644 docs/tenant-governance.md create mode 100644 intelligence-migrations/0034_tenant_governance.sql create mode 100644 services/tenant_governance.py create mode 100644 tests/test_tenant_governance.py create mode 100644 tests/test_tenant_governance_worker.mjs create mode 100644 worker/tenant-governance-d1.mjs diff --git a/docs/tenant-governance.md b/docs/tenant-governance.md new file mode 100644 index 00000000..336658ae --- /dev/null +++ b/docs/tenant-governance.md @@ -0,0 +1,76 @@ +# Tenant governance + +`TENANT_GOVERNANCE_ENABLED` is off by default. Empty values are off. Invalid values +produce one warning without logging the value and are treated as off. When disabled, +key controls, budgets, routing and usage keep their existing behaviour; the governance +administration endpoints return 404. Legacy principals keep that behaviour when enabled. + +The server supplies a verified `TenantContext` through a registered `tenant_resolver` +and an organisation role through `membership_role(principal_id, org_id)`. The default +resolver uses the enterprise legacy authority; the default membership role grants no +permissions. Request headers and JSON fields cannot choose an organisation or role. + +## Grants and permissions + +The key grant, organisation grant and team grant must all allow a model or tool. +A missing ancestor row or a null grant adds no restriction. An empty array denies +everything at that level. Patterns are case insensitive; `*` matches any characters. +The same model intersection applies to automatic route candidates and model-free +provider calls. A provider call without a model requires a whole-provider grant. + +Organisation administrators can edit grants and budgets and read workspace usage. +Billing members can read budgets and workspace usage. Members can read their own +workspace usage. These endpoints cannot edit roles or memberships. + +`GET /admin/organisations//governance` and +`GET /admin/organisations//teams//governance` require an administrator +dashboard session and an appropriate organisation membership. Their `PUT` counterparts +also require CSRF and the quoted revision in `If-Match`. Missing revisions return 428; +stale revisions return 412. Responses expose the successor revision as `ETag`. + +Example policy body with governance enabled: + +```json +{"models":["openai:*"],"tools":["search"],"daily":1000000,"monthly":20000000} +``` + +Budget values are non-negative integer micro-USD, below 9,007,199,254,740,991. +`PUT` replaces the policy: omitted fields become null. It does not add to a grant. +Audits contain identifiers, revision, operation kind and time, without policy content. + +## Monetary reservations and usage + +An enabled workspace admission reserves the estimate against the key, organisation +and team together. Daily periods use UTC dates; monthly periods use UTC calendar +months. A zero budget denies admission. Existing key spending seeds its period +baseline. Unpriced requests return 503 `unpriced_reservation`. An ancestor limit +returns 429 `tenant_budget_exceeded` with the rejecting level; a key limit keeps +`budget_exceeded`. Rejected admission creates no reservation or component. + +Measured completion settles all components atomically. Unknown cost retains the +conservative estimate at every level, including across period changes; holds do +not expire automatically. Pre-dispatch cancellation can release a reservation. +Settlement is idempotent and cannot replace a settled cost with a different amount. +Actual cost can exceed the admission estimate; configure conservative pricing and +output bounds. Estimates and measured gateway costs are not provider invoices. + +`GET /v1/usage` includes `workspace` only for enabled workspace callers. Its history, +model totals and daily totals use that verified workspace, without reading unscoped +legacy history. Members see their own records; administrators and billing members +see workspace totals plus organisation and selected-team totals. Model rows identify +the actual model, provider and price basis. Results have at most 200 model groups +and 90 daily groups. Usage and audits contain no prompts or response content. + +Apply `0034_tenant_governance.sql` before enabling governance. The migration creates +only new tables and indexes and can be rehearsed against existing rows. A missing +table, column or unavailable authority returns JSON 503 +`tenant_governance_unavailable` before admission. No permissive fallback is used. + +The D1 domain is mounted only behind the authenticated private dispatcher at +`http://intelligence.internal/v1/tenant-governance`. Native admission uses +`createGovernanceLifecycle` with verified tenant and key decisions, integer price +estimates, dispatch and finalization hooks. Flask uses the same domain through +`D1GovernanceStore`; `SQLiteGovernanceStore` supports explicitly injected connection +factories and database transactions. Neither adapter applies migrations automatically. +Storage failure prevents new admissions; ambiguous settlement leaves the hold in place. +Retention and any reconciliation of unknown holds require an operator policy. diff --git a/intelligence-migrations/0034_tenant_governance.sql b/intelligence-migrations/0034_tenant_governance.sql new file mode 100644 index 00000000..9afd1048 --- /dev/null +++ b/intelligence-migrations/0034_tenant_governance.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS tenant_governance_policies ( + org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', revision INTEGER NOT NULL DEFAULT 0, + models TEXT, tools TEXT, daily INTEGER CHECK (daily IS NULL OR daily >= 0), + monthly INTEGER CHECK (monthly IS NULL OR monthly >= 0), PRIMARY KEY (org_id, team_id) +); +CREATE TABLE IF NOT EXISTS tenant_governance_reservations ( + id TEXT PRIMARY KEY, principal_id TEXT NOT NULL, org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', + day TEXT NOT NULL, month TEXT NOT NULL, estimate INTEGER NOT NULL CHECK (estimate >= 0), + charged INTEGER, state TEXT NOT NULL CHECK (state IN ('reserved','dispatched','unknown','settled')), + revision INTEGER NOT NULL DEFAULT 0, provider TEXT, model TEXT, price_basis TEXT, + created_at TEXT NOT NULL, operation_id TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS tenant_governance_components ( + reservation_id TEXT NOT NULL REFERENCES tenant_governance_reservations(id), + level TEXT NOT NULL, scope_org TEXT NOT NULL, scope_id TEXT NOT NULL, + period_kind TEXT NOT NULL, period TEXT NOT NULL, limit_units INTEGER, + PRIMARY KEY (reservation_id, level, scope_org, scope_id, period_kind) +); +CREATE INDEX IF NOT EXISTS tenant_governance_component_scope + ON tenant_governance_components(level, scope_org, scope_id, period_kind, period); +CREATE INDEX IF NOT EXISTS tenant_governance_usage_scope + ON tenant_governance_reservations(org_id, team_id, principal_id, day); +CREATE TABLE IF NOT EXISTS tenant_governance_baselines ( + principal_id TEXT NOT NULL, period TEXT NOT NULL, amount INTEGER NOT NULL, + PRIMARY KEY (principal_id, period) +); +CREATE TABLE IF NOT EXISTS tenant_governance_audit ( + operation_id TEXT PRIMARY KEY, actor TEXT NOT NULL, org_id TEXT NOT NULL, team_id TEXT NOT NULL DEFAULT '', + revision INTEGER NOT NULL, kind TEXT NOT NULL, at TEXT NOT NULL +); diff --git a/package.json b/package.json index c66e4a8f..934ee52b 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "ruff check . --select E9,F63,F7,F82", "typecheck": "mypy --ignore-missing-imports --follow-imports=skip --disable-error-code import-untyped --disable-error-code var-annotated --disable-error-code arg-type --disable-error-code assignment --disable-error-code operator app.py routes services providers streaming", "test": "python -m pytest -q", - "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs", + "test:worker": "node --test tests/test_cloudflare_worker.mjs tests/test_cascades_worker.mjs tests/test_cascades_dashboard.mjs tests/test_image_quality_worker.mjs tests/test_secret_scan_worker.mjs tests/test_secret_firewall_worker.mjs tests/test_secret_firewall_coverage_worker.mjs tests/test_roleplay_compaction_budget_worker.mjs tests/test_roleplay_compaction_model_worker.mjs tests/test_roleplay_stream_validation_worker.mjs tests/test_aihubmix_worker.mjs tests/test_gpt_image_moderation_worker.mjs tests/test_dashboard_scripts.mjs tests/test_operations_dashboard.mjs tests/test_opencode_reasoning_worker.mjs tests/test_opencode_thinking_worker.mjs tests/test_nanogpt_reasoning_worker.mjs tests/test_nanogpt_speed_routing_worker.mjs tests/test_sse_heartbeat_worker.mjs tests/test_roleplay_worker.mjs tests/test_roleplay_config_worker.mjs tests/test_roleplay_glm_failover_worker.mjs tests/test_roleplay_intelligence_worker.mjs tests/test_roleplay_memory_worker.mjs tests/test_roleplay_compatibility_worker.mjs tests/test_roleplay_history_reconciliation_worker.mjs tests/test_roleplay_output_contract_analysis.mjs tests/test_roleplay_output_contract_cleanup_worker.mjs tests/test_roleplay_output_contract_nonstream_worker.mjs tests/test_roleplay_output_contract_worker.mjs tests/test_roleplay_performance_worker.mjs tests/test_roleplay_quality_routing_worker.mjs tests/test_roleplay_reasoning_worker.mjs tests/test_roleplay_session_storage_worker.mjs tests/test_roleplay_transport_failure_worker.mjs tests/test_roleplay_unlimited_worker.mjs tests/test_roleplay_workbench_worker.mjs tests/test_roleplay_candidate_context_worker.mjs tests/test_roleplay_operator_worker.mjs tests/test_workbench_stream.mjs tests/test_intelligence_worker.mjs tests/test_intelligence_d1_worker.mjs tests/test_intelligence_auth_worker.mjs tests/test_intelligence_outbound_worker.mjs tests/test_intelligence_operator.mjs tests/test_control_users_worker.mjs tests/test_control_audit_worker.mjs tests/test_access_sso_worker.mjs tests/test_d1_schema_worker.mjs tests/test_shadow_eval_worker.mjs tests/test_auto_routes_worker.mjs tests/test_ai_outbound_worker.mjs tests/test_knowledge_*.mjs tests/test_control_state_worker.mjs tests/test_route_health_worker.mjs tests/test_gateway_mcp_worker.mjs tests/test_usage_ledger_worker.mjs tests/test_media_outbound_worker.mjs tests/test_media_files_worker.mjs tests/test_media_jobs_worker.mjs tests/test_tool_repair_worker.mjs tests/test_prometheus_worker.mjs tests/test_evaluation_noise_floor_worker.mjs tests/test_prompt_templates_worker.mjs tests/test_config_snapshots_worker.mjs tests/test_managed_idempotency_worker.mjs tests/test_wave3_hub_worker.mjs tests/test_wave3_transport_worker.mjs tests/test_admission_leases_worker.mjs tests/test_edge_request_metrics_worker.mjs tests/test_request_cancellation_worker.mjs tests/test_mcp_contract_drift_worker.mjs tests/test_zero_content_retention_worker.mjs tests/test_prompt_cache_cost_worker.mjs tests/test_config_revision_sync_worker.mjs tests/test_gateway_alerts_worker.mjs tests/test_session_tiers_worker.mjs tests/test_usage_reservations_worker.mjs tests/test_shared_generation_cache_worker.mjs tests/test_deferred_tools_worker.mjs tests/test_generation_deadline_worker.mjs tests/test_wave4_worker.mjs tests/test_responses_state_worker.mjs tests/test_gateway_batches_worker.mjs tests/test_semantic_generation_cache_worker.mjs tests/test_canary_traffic_worker.mjs tests/test_context_pages_worker.mjs tests/test_observability_exports_worker.mjs tests/test_prompt_injection_worker.mjs tests/test_pii_rehydration_worker.mjs tests/test_wave5_worker.mjs tests/test_usage_receipts_worker.mjs tests/test_learned_cooldown_worker.mjs tests/test_stream_cost_breaker_worker.mjs tests/test_realtime_worker.mjs tests/test_latency_slo_worker.mjs tests/test_context_canary_worker.mjs tests/test_enterprise_contracts_worker.mjs tests/test_wave6_worker.mjs tests/test_tenant_governance_worker.mjs", "db:migrate": "python scripts/validate_sqlite_schema.py --apply", "db:validate": "python scripts/validate_sqlite_schema.py", "security:secrets": "python scripts/check_static_secrets.py", diff --git a/routes/usage.py b/routes/usage.py index 66b99489..8cd57ab3 100644 --- a/routes/usage.py +++ b/routes/usage.py @@ -2,6 +2,7 @@ administrator's per-key controls.""" import logging +import re from datetime import datetime, timedelta, timezone from flask import g, jsonify, render_template, request @@ -10,7 +11,7 @@ from middleware.rate_limit_headers import enabled as rate_limit_headers_enabled, usage_snapshot from request_validation import json_object_body from route_helpers import api_authenticate_only, login_required -from services import key_controls, usage_ledger, usage_store +from services import key_controls, usage_ledger, usage_store, tenant_governance from services.auth_service import AuthService from services.budget_service import BudgetService @@ -65,6 +66,7 @@ def usage_history(principal, days: int, *, by_principal: bool = False) -> dict: def register_usage_routes(app, csrf) -> None: + register_governance_routes(app) @app.route("/v1/usage", methods=["GET", "OPTIONS"]) @csrf.exempt @api_authenticate_only(required_scope="models") @@ -73,6 +75,16 @@ def api_usage(): user = g.authenticated_user principal = str(user.get("username") or user.get("id")) controls = key_controls.public(user) + since, until = _range(_days()) + workspace = tenant_governance.workspace_usage(user, since=since, until=until) + if workspace: + scoped = workspace["workspace"] + return jsonify({"object": "usage", "principal": principal, "key_prefix": user.get("api_key_prefix"), + "controls": {name: controls[name] for name in ("allowed_models", "allowed_ips", "expires_at")}, + "budget": {"daily_budget_usd": user.get("daily_budget_usd"), + "monthly_budget_usd": user.get("monthly_budget_usd")}, + "history_available": True, "range": scoped["range"], "daily": scoped["daily"], "models": scoped["models"], + "totals": {key: scoped[key] for key in ("spent_micro_usd", "holds_micro_usd", "requests")}, **workspace}) return jsonify({ **({"rate_limits": usage_snapshot(user, request.remote_addr)} if rate_limit_headers_enabled() else {}), @@ -82,6 +94,7 @@ def api_usage(): "budget": BudgetService.status(user), "controls": {name: controls[name] for name in ("allowed_models", "allowed_ips", "expires_at")}, **usage_history(principal, _days()), + **workspace, }) @app.route("/usage") @@ -123,3 +136,65 @@ def update_key_controls(username: str): except APIError as error: return jsonify({"status": "error", "message": error.client_message}), error.status_code return jsonify({"status": "success", "user": user}) + + +def register_governance_routes(app): + if app.extensions.get("tenant_governance_routes_registered"): + return + app.extensions["tenant_governance_routes_registered"] = True + + def disabled_governance(): + if request.path.startswith("/admin/organisations/") and request.path.endswith("/governance"): + if not tenant_governance.enabled(): + return jsonify({"error": {"code": "not_found"}}), 404 + return None + + app.before_request_funcs.setdefault(None, []).insert(0, disabled_governance) + + @app.after_request + def governance_budget_envelope(response): + if tenant_governance.enabled() and response.status_code == 429 and response.is_json: + payload = response.get_json(silent=True) + if isinstance(payload, dict) and payload.get("error") == "tenant_budget_exceeded" and isinstance(payload.get("budget"), dict): + payload["error"] = {"code": "tenant_budget_exceeded", "level": payload["budget"].get("level")} + response.set_data(app.json.dumps(payload) + "\n") + return response + + @app.errorhandler(tenant_governance.GovernanceError) + def governance_error(error): + return jsonify(error.payload), error.status + + def governance(org_id, team_id=None): + user = AuthService.get_current_user() or {} + if not user.get("is_admin"): + raise tenant_governance.GovernanceError("administrator_required", 403) + tenant_governance._scope(org_id, team_id) + principal = str(user.get("username") or user.get("id") or "") + role = tenant_governance.membership_role(principal, org_id) + permission = "edit" if request.method == "PUT" else "budgets" + if not tenant_governance.permitted(role, permission): + raise tenant_governance.GovernanceError("tenant_role_denied", 403) + values = {"org_id": org_id, "team_id": team_id} + if request.method == "PUT": + match = request.headers.get("If-Match") + if match is None: + raise tenant_governance.GovernanceError("revision_required", 428) + if not re.fullmatch(r'"(?:0|[1-9][0-9]{0,15})"', match): + raise tenant_governance.GovernanceError("revision_stale", 412) + values.update(actor=principal, revision=int(match[1:-1]), + policy=tenant_governance.validate_policy(json_object_body())) + result = tenant_governance.store().call("put" if request.method == "PUT" else "get", **values) + response = jsonify(result) + response.headers["ETag"] = f'"{result["policy"]["revision"]}"' + response.headers["Cache-Control"] = "no-store" + return response + + @app.route("/admin/organisations//governance", methods=["GET", "PUT"]) + @login_required + def organisation_governance(org_id): + return governance(org_id) + + @app.route("/admin/organisations//teams//governance", methods=["GET", "PUT"]) + @login_required + def team_governance(org_id, team_id): + return governance(org_id, team_id) diff --git a/services/budget_service.py b/services/budget_service.py index 4011fe32..022aa65a 100644 --- a/services/budget_service.py +++ b/services/budget_service.py @@ -55,7 +55,8 @@ def limits(user: Mapping[str, Any]) -> tuple[Optional[float], Optional[float]]: def budgeted(user: Mapping[str, Any]) -> bool: - return any(limit is not None for limit in limits(user)) + from services.tenant_governance import workspace_context + return workspace_context(user) is not None or any(limit is not None for limit in limits(user)) @dataclass @@ -201,6 +202,10 @@ def _spent(cls, state: _Principal, period: dict) -> tuple[float, float, float]: @classmethod def check_and_reserve(cls, user: Mapping[str, Any], estimate_usd: Optional[float], now: Optional[datetime] = None) -> BudgetDecision: + from services.tenant_governance import reserve_budget + governance = reserve_budget(user, estimate_usd, now, cls) + if governance is not None: + return governance daily, monthly = limits(user) if daily is None and monthly is None: return BudgetDecision(True) @@ -278,6 +283,9 @@ def _reserve_durable(cls, user, estimate, now=None): @classmethod def mark_dispatched(cls, reservation): """Persist before provider submission; failure must prevent submission.""" + from services.tenant_governance import dispatch + if dispatch(reservation): + return if not reservation or not reservation_store.enabled(): return store = reservation_store.get_store() @@ -290,6 +298,9 @@ def mark_dispatched(cls, reservation): @classmethod def complete(cls, reservation, row, *, cached=False): """Replace a hold only with measured cost; uncertain completion retains it.""" + from services.tenant_governance import complete + if complete(reservation, row, cached=cached): + return if not reservation or not reservation_store.enabled(): return store = reservation_store.get_store() @@ -314,6 +325,9 @@ def settle(cls, reservation: Optional[str], *, before_dispatch=False) -> None: """Release an in-flight estimate; the settled cost arrives through `record_cost`.""" if not reservation: return + from services.tenant_governance import complete + if complete(reservation, {}, before_dispatch=before_dispatch): + return if reservation_store.enabled(): if before_dispatch: store = reservation_store.get_store() @@ -336,6 +350,8 @@ def record_cost(cls, row: Mapping[str, Any]) -> None: Call before handing the row to the ledger, which then reports it as stored or dropped exactly once. """ + from services.tenant_governance import capture_cost + capture_cost(row) with cls._lock: state = cls._principals.setdefault(str(row["principal"]), _Principal()) _add(state.pending, row) diff --git a/services/intelligence_policy.py b/services/intelligence_policy.py index 174e2726..cec1fbab 100644 --- a/services/intelligence_policy.py +++ b/services/intelligence_policy.py @@ -189,12 +189,14 @@ def validate_policy(raw): def eligible(candidate, allow_paid, config, *, model_status=None): provider = candidate["model"].split(":", 1)[0] model_status = model_status or ModelRegistry.get_model_status - return ( + allowed = ( all(candidate[key] for key in ("enabled", "entitled", "privacy_allowed")) and (candidate["billing"] != "payg" or allow_paid) and model_status(candidate["model"]) != "disabled" and get_adapter(provider, config["API_BASE_URLS"]) is not None ) + from services.tenant_governance import eligibility_allowed + return eligibility_allowed(allowed, candidate["model"]) def input_reservation(candidate, request): diff --git a/services/key_controls.py b/services/key_controls.py index 633db632..017ee8b0 100644 --- a/services/key_controls.py +++ b/services/key_controls.py @@ -212,20 +212,27 @@ def model_allowed(user: Mapping[str, Any], model: Any) -> bool: """Whether a model ID matches the key's allowlist; ``*`` matches any characters.""" patterns = model_patterns(user) if patterns is None: - return True - if not isinstance(model, str) or not model.strip(): - return False - candidate = model.strip().lower() - return any(_glob(pattern.lower()).match(candidate) for pattern in patterns) + allowed = True + elif not isinstance(model, str) or not model.strip(): + allowed = False + else: + candidate = model.strip().lower() + allowed = any(_glob(pattern.lower()).match(candidate) for pattern in patterns) + from services.tenant_governance import grant_allowed + return grant_allowed(allowed, model, user=user) def provider_allowed(user: Mapping[str, Any], provider: str) -> bool: """Whether a request that names no model may reach a provider namespace.""" patterns = model_patterns(user) if patterns is None: - return True - provider = provider.strip().lower() - return any(pattern in {"*", f"{provider}:*"} for pattern in patterns) + allowed = True + candidate = provider.strip().lower() if isinstance(provider, str) else "" + else: + candidate = provider.strip().lower() + allowed = any(pattern in {"*", f"{candidate}:*"} for pattern in patterns) + from services.tenant_governance import grant_allowed + return grant_allowed(allowed, f"{candidate}:*", user=user) def expires_at(user: Mapping[str, Any]) -> Optional[datetime]: diff --git a/services/tenant_governance.py b/services/tenant_governance.py new file mode 100644 index 00000000..3b128c75 --- /dev/null +++ b/services/tenant_governance.py @@ -0,0 +1,485 @@ +"""Intersected workspace grants and durable multi-level monetary reservations.""" +from __future__ import annotations + +import json +import logging +import os +import re +import sqlite3 +import uuid +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from decimal import Decimal, ROUND_CEILING +from functools import lru_cache + +from flask import current_app, g, has_app_context, has_request_context + +from error_handlers import APIError +from services import control_state_d1 +from services.enterprise_contract import AuthorityOperation, MAX_INTEGER, TenantContext, legacy_tenant + +logger = logging.getLogger(__name__) +PERMISSIONS = { + "admin": frozenset({"edit", "budgets", "usage", "all_usage"}), + "billing": frozenset({"budgets", "usage", "all_usage"}), + "member": frozenset({"usage"}), +} +TABLES = ("tenant_governance_policies", "tenant_governance_reservations", + "tenant_governance_components", "tenant_governance_baselines", "tenant_governance_audit") +SCHEMA_QUERIES = ( + "SELECT org_id,team_id,revision,models,tools,daily,monthly FROM tenant_governance_policies LIMIT 0", + "SELECT id,principal_id,org_id,team_id,day,month,estimate,charged,state,revision,provider,model,price_basis,created_at,operation_id FROM tenant_governance_reservations LIMIT 0", + "SELECT reservation_id,level,scope_org,scope_id,period_kind,period,limit_units FROM tenant_governance_components LIMIT 0", + "SELECT principal_id,period,amount FROM tenant_governance_baselines LIMIT 0", + "SELECT operation_id,actor,org_id,team_id,revision,kind,at FROM tenant_governance_audit LIMIT 0", +) +POLICY_FIELDS = frozenset({"models", "tools", "daily", "monthly"}) + + +class GovernanceError(APIError): + def __init__(self, code="tenant_governance_unavailable", status=503, *, level=None, period=None): + self.code, self.status, self.level, self.period = code, status, level, period + detail = {"code": code} + if level: + detail["level"] = level + if period: + detail["period"] = period + super().__init__("The workspace governance operation could not be completed.", status, {"error": detail}) + + +@lru_cache(maxsize=1) +def _warn_once(): + logger.warning("Invalid TENANT_GOVERNANCE_ENABLED; tenant governance disabled") + + +def enabled(env=None): + value = (os.environ if env is None else env).get("TENANT_GOVERNANCE_ENABLED", "") + flag = value.strip().lower() if isinstance(value, str) else "invalid" + if flag in {"", "false", "0", "off", "no"}: + return False + if flag in {"true", "1", "on", "yes"}: + return True + _warn_once() + return False + + +def permitted(role, permission): + return permission in PERMISSIONS.get(role, ()) + + +def _legacy_resolver(user): + context = TenantContext(str(user.get("username") or user.get("id") or "")) + return legacy_tenant(AuthorityOperation(context, context.principal_id, 0, "governance.resolve")) + + +def _no_role(principal_id, org_id): + return None + + +@dataclass(frozen=True) +class GovernanceCollaborators: + tenant_resolver: object = _legacy_resolver + membership_role: object = _no_role + store: object = None + + +_collaborators = GovernanceCollaborators() + + +def register_governance_collaborators(*, tenant_resolver=_legacy_resolver, + membership_role=_no_role, store=None, app=None): + """Register fixed trusted collaborators; caller metadata never selects a workspace.""" + if not callable(tenant_resolver) or not callable(membership_role): + raise TypeError("Named tenant resolver and membership role functions required") + if store is not None and not callable(getattr(store, "call", None)): + raise TypeError("Governance store call interface required") + collaborators = GovernanceCollaborators(tenant_resolver, membership_role, store) + if app is not None: + app.extensions["tenant_governance"] = collaborators + else: + global _collaborators + _collaborators = collaborators + return collaborators + + +def collaborators(): + return current_app.extensions.get("tenant_governance", _collaborators) if has_app_context() else _collaborators + + +def workspace_context(user=None): + if not enabled(): + return None + user = user if user is not None else (getattr(g, "authenticated_user", None) if has_request_context() else None) + if user is None: + return None + try: + context = collaborators().tenant_resolver(user) + if type(context) is not TenantContext or context.principal_id != str(user.get("username") or user.get("id") or ""): + raise GovernanceError("tenant_scope_denied", 403) + except GovernanceError: + raise + except Exception: + raise GovernanceError("tenant_scope_denied", 403) from None + return context if context.org_id is not None else None + + +def context_dict(context): + return asdict(context) + + +def membership_role(principal_id, org_id): + return collaborators().membership_role(principal_id, org_id) + + +class D1GovernanceStore: + def call(self, operation, **values): + if not control_state_d1.using_d1(): + raise GovernanceError() + try: + result = control_state_d1.call("tenant_governance", operation, version=1, rpc=True, **values) + except control_state_d1.intelligence_d1_store.PrivateIntelligenceError as error: + raise GovernanceError(error.code, error.status) from None + except Exception: + raise GovernanceError() from None + if not isinstance(result, dict) or result.get("version") != 1: + raise GovernanceError() + decision = result.get("decision") + if isinstance(decision, dict) and decision.get("allowed") is False: + raise GovernanceError(decision["code"], decision["status"], + level=decision.get("level"), period=decision.get("period")) + if result.get("error"): + failure = result["error"] + code = failure.get("code", "tenant_governance_unavailable") + status = {"tenant_budget_exceeded": 429, "budget_exceeded": 429, + "revision_stale": 412, "reservation_conflict": 409}.get(code, 503) + raise GovernanceError(code, status, level=failure.get("level"), period=failure.get("period")) + return {key: value for key, value in result.items() if key != "version"} + + +def store(): + return collaborators().store or D1GovernanceStore() + + +def validate_policy(value): + if not isinstance(value, dict) or set(value) - POLICY_FIELDS: + raise GovernanceError("invalid_governance_policy", 400) + result = {key: value.get(key) for key in POLICY_FIELDS} + for key in ("daily", "monthly"): + amount = result[key] + if amount is not None and (type(amount) is not int or not 0 <= amount < MAX_INTEGER): + raise GovernanceError("invalid_governance_policy", 400) + for key in ("models", "tools"): + entries = result[key] + if entries is not None and (not isinstance(entries, list) or len(entries) > 64 or + any(not isinstance(entry, str) or re.fullmatch(r"[A-Za-z0-9._:/+@*\-]{1,256}", entry) is None for entry in entries)): + raise GovernanceError("invalid_governance_policy", 400) + return result + + +def _matches(patterns, candidate): + if patterns is None: + return True + if not isinstance(candidate, str) or not candidate.strip(): + return False + return any(re.fullmatch(".*".join(re.escape(part) for part in pattern.lower().split("*")), + candidate.strip().lower()) for pattern in patterns) + + +def grant_allowed(existing_allowed, candidate, *, user=None, kind="models", context=None): + """Every configured ancestor intersects the existing key decision.""" + if not enabled(): + return existing_allowed + context = context if context is not None else workspace_context(user) + if context is None or context.org_id is None: + return existing_allowed + if kind not in {"models", "tools"}: + raise ValueError("Unknown grant kind") + policies = store().call("policies", context=context_dict(context))["policies"] + return existing_allowed and all(_matches(policy[kind], candidate) for policy in policies) + + +def eligibility_allowed(existing_allowed, model): + if not enabled() or not has_request_context(): + return existing_allowed + user = getattr(g, "authenticated_user", None) + if workspace_context(user) is None: + return existing_allowed + from services.key_controls import model_allowed + key_allowed = model_allowed(user, model) + return existing_allowed and key_allowed + + +def units(value): + if isinstance(value, bool) or value is None: + raise GovernanceError("unpriced_reservation", 503) + try: + number = Decimal(str(value)) * 1_000_000 + if not number.is_finite() or number < 0 or number >= MAX_INTEGER: + raise ValueError + return int(number.to_integral_value(rounding=ROUND_CEILING)) + except (ValueError, ArithmeticError): + raise GovernanceError("unpriced_reservation", 503) from None + + +def reserve_budget(user, estimate_usd, now, budget_service): + from services.budget_service import BudgetDecision, BudgetUnavailable, limits, periods + try: + context = workspace_context(user) + if context is None: + return None + amount = units(estimate_usd) + period = periods(now) + daily, monthly = limits(user) + base_day = base_month = 0 + if daily is not None or monthly is not None: + state = budget_service._state(context.principal_id) + budget_service._refresh(context.principal_id, state, period) + with budget_service._lock: + day, month, inflight = budget_service._spent(state, period) + base_day, base_month = units(day + inflight), units(month + inflight) + identity = "tg_" + uuid.uuid4().hex + store().call("reserve", id=identity, context=context_dict(context), amount=amount, + day=period["day"], month=period["month"], + key_daily=None if daily is None else units(daily), + key_monthly=None if monthly is None else units(monthly), + base_day=base_day, base_month=base_month) + if has_request_context(): + # Accounting clears usage_context before recording its final row. + g.tenant_governance_reservation = identity + g.tenant_governance_principal = context.principal_id + return BudgetDecision(True, reservation=identity) + except (GovernanceError, BudgetUnavailable) as error: + if isinstance(error, BudgetUnavailable): + error = GovernanceError() + details = {"level": error.level, "period": error.period} if error.level else {} + return BudgetDecision(False, error=error.code, status_code=error.status, + message="The workspace budget could not admit this request.", details=details) + + +def governance_reservation(identity): + return isinstance(identity, str) and identity.startswith("tg_") + + +def dispatch(identity): + if not governance_reservation(identity): + return False + store().call("dispatch", id=identity) + return True + + +def complete(identity, row, *, before_dispatch=False, cached=False): + if not governance_reservation(identity): + return False + model = row.get("selected_model") + model = model if isinstance(model, str) and re.fullmatch(r"[A-Za-z0-9._:/+@\-]{1,256}", model) else None + cost = row.get("cost_usd") + measured = cost is not None and row.get("cost_basis") == "usage" and not cached + store().call("settle", id=identity, cost=0 if before_dispatch or cached else units(cost) if measured else None, + provider=model.split(":", 1)[0] if model and ":" in model else None, model=model, + price_basis="released" if before_dispatch else "cache" if cached else row.get("cost_basis"), + before_dispatch=before_dispatch) + return True + + +def capture_cost(row): + if not has_request_context(): + return + identity = getattr(g, "tenant_governance_reservation", None) + if governance_reservation(identity) and row.get("principal") == getattr(g, "tenant_governance_principal", None): + complete(identity, row, cached=row.get("cost_basis") == "cache") + + +def workspace_usage(user, *, since="0000-01-01", until="9999-12-31"): + context = workspace_context(user) + if context is None: + return {} + role = membership_role(context.principal_id, context.org_id) + if not permitted(role, "usage"): + raise GovernanceError("tenant_role_denied", 403) + return {"workspace": store().call("usage", context=context_dict(context), role=role, since=since, until=until)} + + +def _scope(org_id, team_id): + try: + TenantContext("scope", org_id, team_id) + except (ValueError, TypeError): + raise GovernanceError("invalid_tenant_scope", 400) from None + if org_id is None: + raise GovernanceError("invalid_tenant_scope", 400) + return org_id, team_id or "" + + +def _policy(row): + result = {"revision": row["revision"], "models": json.loads(row["models"]) if row["models"] is not None else None, + "tools": json.loads(row["tools"]) if row["tools"] is not None else None, + "daily": row["daily"], "monthly": row["monthly"]} + try: + validate_policy({key: result[key] for key in POLICY_FIELDS}) + except GovernanceError: + raise GovernanceError() from None + return result + + +class SQLiteGovernanceStore: + """Injected SQLite adapter. Migration application belongs to the operator.""" + def __init__(self, connect): + self.connect = connect + + @contextmanager + def transaction(self): + db = self.connect() + db.row_factory = sqlite3.Row + try: + db.execute("BEGIN IMMEDIATE") + for query in SCHEMA_QUERIES: + db.execute(query) + yield db + db.commit() + except GovernanceError: + db.rollback() + raise + except (sqlite3.Error, OSError, ValueError, TypeError, KeyError): + db.rollback() + raise GovernanceError() from None + finally: + db.close() + + def call(self, operation, **values): + with self.transaction() as db: + if operation in {"get", "put"}: + return self._admin(db, operation, values) + if operation == "policies": + context = TenantContext(**values["context"]) + return {"policies": self._policies(db, context)} + if operation == "reserve": + return self._reserve(db, values) + if operation in {"settle", "dispatch"}: + return self._settle(db, operation, values) + if operation == "usage": + return self._usage(db, values) + raise GovernanceError("invalid_governance_operation", 400) + + def _policies(self, db, context): + return [_policy(row) for row in db.execute( + "SELECT * FROM tenant_governance_policies WHERE org_id=? AND (team_id='' OR team_id=?)", + (context.org_id, context.team_id or ""))] + + def _admin(self, db, operation, values): + org, team = _scope(values["org_id"], values.get("team_id")) + row = db.execute("SELECT * FROM tenant_governance_policies WHERE org_id=? AND team_id=?", (org, team)).fetchone() + revision = row["revision"] if row else 0 + if operation == "get": + return {"policy": _policy(row) if row else {"revision": 0, **dict.fromkeys(POLICY_FIELDS)}} + policy = validate_policy(values["policy"]) + if type(values["revision"]) is not int or values["revision"] != revision: + raise GovernanceError("revision_stale", 412) + actor = TenantContext(values["actor"]).principal_id + db.execute("""INSERT INTO tenant_governance_policies VALUES (?,?,?,?,?,?,?) + ON CONFLICT(org_id,team_id) DO UPDATE SET revision=excluded.revision,models=excluded.models, + tools=excluded.tools,daily=excluded.daily,monthly=excluded.monthly""", + (org, team, revision + 1, None if policy["models"] is None else json.dumps(policy["models"]), + None if policy["tools"] is None else json.dumps(policy["tools"]), policy["daily"], policy["monthly"])) + db.execute("INSERT INTO tenant_governance_audit VALUES (?,?,?,?,?,?,?)", + (uuid.uuid4().hex, actor, org, team, revision + 1, "policy", datetime.now(timezone.utc).isoformat())) + return {"policy": {"revision": revision + 1, **policy}} + + def _components(self, db, context, values): + components = [] + scopes = [("key", "", context.principal_id, values["key_daily"], values["key_monthly"])] + for team in [""] + ([context.team_id] if context.team_id else []): + row = db.execute("SELECT daily,monthly FROM tenant_governance_policies WHERE org_id=? AND team_id=?", + (context.org_id, team)).fetchone() + scopes.append(("team" if team else "organisation", context.org_id, team or context.org_id, + row["daily"] if row else None, row["monthly"] if row else None)) + for level, org, scoped, daily, monthly in scopes: + for kind, period, limit in (("daily", values["day"], daily), ("monthly", values["month"], monthly)): + components.append((level, org, scoped, kind, period, limit)) + return components + + def _reserve(self, db, values): + context = TenantContext(**values["context"]) + if context.org_id is None: + raise GovernanceError("tenant_scope_denied", 403) + amount = values["amount"] + if type(amount) is not int or not 0 <= amount < MAX_INTEGER: + raise GovernanceError("unpriced_reservation", 503) + if db.execute("SELECT id FROM tenant_governance_reservations WHERE id=?", (values["id"],)).fetchone(): + raise GovernanceError("reservation_conflict", 409) + components = self._components(db, context, values) + for period, baseline in ((values["day"], values["base_day"]), (values["month"], values["base_month"])): + db.execute("INSERT OR IGNORE INTO tenant_governance_baselines VALUES (?,?,?)", (context.principal_id, period, baseline)) + for level, org, scoped, kind, period, limit in components: + if limit is None: + continue + total = db.execute("""SELECT COALESCE(SUM(CASE WHEN r.state!='settled' THEN r.estimate + WHEN c.period=? THEN r.charged ELSE 0 END),0) FROM tenant_governance_components c + JOIN tenant_governance_reservations r ON r.id=c.reservation_id + WHERE c.level=? AND c.scope_org=? AND c.scope_id=? AND c.period_kind=?""", + (period, level, org, scoped, kind)).fetchone()[0] + if level == "key": + total += db.execute("SELECT amount FROM tenant_governance_baselines WHERE principal_id=? AND period=?", + (context.principal_id, period)).fetchone()[0] + if total >= limit or amount > limit - total: + raise GovernanceError("budget_exceeded" if level == "key" else "tenant_budget_exceeded", 429, + level=level, period=kind) + db.execute("""INSERT INTO tenant_governance_reservations + (id,principal_id,org_id,team_id,day,month,estimate,state,created_at,operation_id) VALUES (?,?,?,?,?,?,?,'reserved',?,?)""", + (values["id"], context.principal_id, context.org_id, context.team_id or "", values["day"], values["month"], + amount, datetime.now(timezone.utc).isoformat(), uuid.uuid4().hex)) + db.executemany("INSERT INTO tenant_governance_components VALUES (?,?,?,?,?,?,?)", + [(values["id"], *component) for component in components]) + return {"id": values["id"]} + + def _settle(self, db, operation, values): + row = db.execute("SELECT * FROM tenant_governance_reservations WHERE id=?", (values["id"],)).fetchone() + if row is None: + raise GovernanceError("reservation_conflict", 409) + if row["state"] in {"settled", "unknown"}: + if (row["state"] == "settled" and operation == "settle" and values.get("cost") is not None + and row["charged"] != values["cost"]): + raise GovernanceError("reservation_conflict", 409) + return {"state": row["state"]} + if operation == "dispatch": + db.execute("UPDATE tenant_governance_reservations SET state='dispatched',revision=revision+1 WHERE id=?", (values["id"],)) + return {"state": "dispatched"} + if values.get("before_dispatch") and row["state"] != "reserved": + raise GovernanceError("reservation_conflict", 409) + cost = values["cost"] + if cost is not None and (type(cost) is not int or not 0 <= cost < MAX_INTEGER): + raise GovernanceError("invalid_governance_cost", 400) + state = "unknown" if cost is None else "settled" + db.execute("""UPDATE tenant_governance_reservations SET state=?,charged=?,provider=?,model=?,price_basis=?, + revision=revision+1 WHERE id=?""", (state, cost, values.get("provider"), values.get("model"), + values.get("price_basis"), values["id"])) + return {"state": state} + + def _usage(self, db, values): + context = TenantContext(**values["context"]) + role = values["role"] + if not permitted(role, "usage"): + raise GovernanceError("tenant_role_denied", 403) + since, until = values.get("since", "0000-01-01"), values.get("until", "9999-12-31") + params = (context.org_id, context.team_id or "", context.team_id or "", + int(permitted(role, "all_usage")), context.principal_id, since, until) + totals = dict(db.execute("""SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd, + COUNT(*) AS requests FROM tenant_governance_reservations WHERE org_id=? AND (?='' OR team_id=?) + AND (?=1 OR principal_id=?) AND day>=? AND day<=?""", params).fetchone()) + models = [dict(row) for row in db.execute("""SELECT provider,model,price_basis,COUNT(*) AS requests, + COALESCE(SUM(charged),0) AS spent_micro_usd FROM tenant_governance_reservations + WHERE org_id=? AND (?='' OR team_id=?) AND (?=1 OR principal_id=?) AND day>=? AND day<=? + GROUP BY provider,model,price_basis ORDER BY model LIMIT 200""", params)] + daily = [dict(row) for row in db.execute("""SELECT day,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND (?='' OR team_id=?) + AND (?=1 OR principal_id=?) AND day>=? AND day<=? GROUP BY day ORDER BY day LIMIT 90""", params)] + result = {"org_id": context.org_id, "team_id": context.team_id, "principal_id": context.principal_id, + "cost_description": "Gateway cost estimates, not invoices", **totals, "models": models, "daily": daily, + "range": {"since": since, "until": until}} + if permitted(role, "all_usage"): + org_total = dict(db.execute("""SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND day>=? AND day<=?""", (context.org_id, since, until)).fetchone()) + result["organisation"] = org_total + result["team"] = totals if context.team_id else None + return result diff --git a/tests/fixtures/secret_firewall_coverage.json b/tests/fixtures/secret_firewall_coverage.json index 423bb4f5..0cbf8396 100644 --- a/tests/fixtures/secret_firewall_coverage.json +++ b/tests/fixtures/secret_firewall_coverage.json @@ -192,6 +192,8 @@ "app.route('/v1/messages/count_tokens', methods=['POST', 'OPTIONS'])" ], "routes/usage.py": [ + "app.route('/admin/organisations//governance', methods=['GET', 'PUT'])", + "app.route('/admin/organisations//teams//governance', methods=['GET', 'PUT'])", "app.route('/usage')", "app.route('/usage/data')", "app.route('/users//controls', methods=['PUT'])", @@ -409,6 +411,7 @@ "worker/sse-heartbeat.mjs", "worker/status-page.mjs", "worker/stream-cost-breaker.mjs", + "worker/tenant-governance-d1.mjs", "worker/upstream-cancellation.mjs", "worker/usage-buckets-d1.mjs", "worker/usage-ledger-d1.mjs", diff --git a/tests/test_tenant_governance.py b/tests/test_tenant_governance.py new file mode 100644 index 00000000..45ba8890 --- /dev/null +++ b/tests/test_tenant_governance.py @@ -0,0 +1,364 @@ +"""Governance policy, transactional quotas and registered HTTP boundaries.""" +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask, g +from flask_wtf import CSRFProtect +from flask_wtf.csrf import generate_csrf + +from services import tenant_governance as governance, key_controls, budget_service +from services.enterprise_contract import TenantContext +from tests.unified_api_test_case import UnifiedApiTestCase + +NOW = datetime(2026, 10, 9, tzinfo=timezone.utc) +USER = {"username": "alice", "daily_budget_usd": 1.0, "allowed_models": "openai:*"} +TENANT = TenantContext("alice", "org1", "team1") + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "true") + monkeypatch.setenv("USAGE_RESERVATIONS_ENABLED", "false") + monkeypatch.setenv("USAGE_LEDGER_ENABLED", "false") + path = tmp_path / "governance.sqlite3" + with sqlite3.connect(path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + store = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(path, timeout=10)) + app = Flask(__name__) + app.secret_key = "synthetic-governance-test-secret" + governance.register_governance_collaborators(app=app, store=store, + tenant_resolver=lambda user: TenantContext(user["username"], "org1", "team1"), + membership_role=lambda principal, org: "admin" if org == "org1" else None) + budget_service.BudgetService.reset() + yield app, store + budget_service.BudgetService.reset() + + +def policy(store, org="org1", team=None, **fields): + return store.call("put", org_id=org, team_id=team, actor="alice", revision=0, + policy={"models": None, "tools": None, "daily": None, "monthly": None, **fields}) + + +@pytest.mark.parametrize("flag", [None, "", "false", "broken"]) +def test_disabled_never_resolves_or_reads_storage(setup, monkeypatch, flag): + app, _ = setup + if flag is None: + monkeypatch.delenv("TENANT_GOVERNANCE_ENABLED") + else: + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", flag) + def forbidden(*args): + pytest.fail("disabled governance touched authority") + governance.register_governance_collaborators(app=app, tenant_resolver=forbidden) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not key_controls.model_allowed(USER, "other:gpt") + assert not budget_service.budgeted({"username": "alice"}) + + +def test_legacy_remains_unrestricted_by_ancestors(setup): + app, store = setup + policy(store, models=[]) + governance.register_governance_collaborators(app=app, store=store) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not governance.workspace_context(USER) + + +def test_grants_intersect_key_team_and_organisation_and_tools(setup): + app, store = setup + policy(store, models=["openai:*"], tools=["search"]) + policy(store, team="team1", models=["openai:gpt"], tools=[]) + with app.test_request_context(): + assert key_controls.model_allowed(USER, "openai:gpt") + assert not key_controls.model_allowed(USER, "openai:other") + assert not key_controls.model_allowed({**USER, "allowed_models": "other:*"}, "openai:gpt") + assert not governance.grant_allowed(True, "search", user=USER, kind="tools") + assert not governance.grant_allowed(False, "search", user=USER, kind="tools") + + +def test_atomic_reject_and_settlement_at_all_levels(setup): + app, store = setup + policy(store, daily=1_000_000, monthly=2_000_000) + policy(store, team="team1", daily=500_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert first.allowed + denied = budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW) + assert not denied.allowed and denied.error == "tenant_budget_exceeded" + assert denied.details["level"] == "team" + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + budget_service.BudgetService.complete(first.reservation, + {"selected_model": "openai:gpt", "cost_usd": 0.1, "cost_basis": "usage", "status": 200}) + assert budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW).allowed + + +def test_unknown_holds_survive_settle_and_disabled_flag(setup, monkeypatch): + app, store = setup + policy(store, daily=500_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + budget_service.BudgetService.complete(first.reservation, {"cost_usd": None, "cost_basis": None}) + budget_service.BudgetService.settle(first.reservation) + assert not budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW).allowed + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "false") + budget_service.BudgetService.settle(first.reservation) + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + + +def test_concurrent_race_has_one_winner_across_connections(setup): + app, store = setup + policy(store, daily=500_000) + def reserve(_): + with app.test_request_context(): + return budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(reserve, range(8))) + assert sum(result.allowed for result in results) == 1 + assert sum(result.error == "tenant_budget_exceeded" for result in results) == 7 + assert store.call("usage", context=governance.context_dict(TENANT), role="admin")["holds_micro_usd"] == 400_000 + + +def test_missing_schema_fails_closed_before_key_reservation(setup): + app, _ = setup + empty = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(":memory:")) + governance.register_governance_collaborators(app=app, store=empty, tenant_resolver=lambda user: TENANT) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + key_controls.model_allowed(USER, "openai:gpt") + assert failure.value.code == "tenant_governance_unavailable" and failure.value.status == 503 + decision = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert decision.error == "tenant_governance_unavailable" and decision.status_code == 503 + assert budget_service.BudgetService._reservations == {} + + +def test_usage_isolation_attribution_and_role_permissions(setup): + app, store = setup + policy(store, daily=1_000_000) + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.2, NOW) + budget_service.BudgetService.complete(first.reservation, + {"selected_model": "openai:actual", "cost_usd": 0.1, "cost_basis": "usage"}) + second = budget_service.BudgetService.check_and_reserve({**USER, "username": "bob"}, 0.2, NOW) + budget_service.BudgetService.complete(second.reservation, + {"selected_model": "other:actual", "cost_usd": 0.15, "cost_basis": "usage"}) + own = store.call("usage", context=governance.context_dict(TENANT), role="member") + all_usage = store.call("usage", context=governance.context_dict(TENANT), role="billing") + foreign = store.call("usage", context=governance.context_dict(TenantContext("alice", "org2")), role="admin") + assert own["spent_micro_usd"] == 100_000 and all_usage["spent_micro_usd"] == 250_000 + assert foreign["spent_micro_usd"] == 0 + assert own["models"][0]["model"] == "openai:actual" and own["models"][0]["provider"] == "openai" + assert own["models"][0]["price_basis"] == "usage" + assert "organisation" not in own and "organisation" in all_usage + assert not governance.permitted("billing", "edit") and not governance.permitted("member", "budgets") + + +def test_registered_admin_routes_csrf_cas_and_no_role_writes(setup, monkeypatch): + app, store = setup + from routes import usage + monkeypatch.setattr(usage.AuthService, "get_current_user", lambda: {"username": "alice", "is_admin": True}) + csrf = CSRFProtect(app) + usage.register_usage_routes(app, csrf) + @app.route("/test-csrf") + def csrf_token(): + return {"token": generate_csrf()} + client = app.test_client() + with client.session_transaction() as session: + session["authenticated"] = True + session["user"] = "alice" + url = "/admin/organisations/org1/governance" + assert client.get(url).status_code == 200 + assert client.put(url, json={}).status_code == 400 + token = client.get("/test-csrf").json["token"] + headers = {"X-CSRFToken": token} + assert client.put(url, json={"daily": 1000}, headers=headers).status_code == 428 + headers["If-Match"] = '"0"' + updated = client.put(url, json={"daily": 1000}, headers=headers) + assert updated.status_code == 200 and updated.headers["ETag"] == '"1"' + assert client.put(url, json={"daily": 2000}, headers=headers).status_code == 412 + headers["If-Match"] = '"1"' + assert client.put(url, json={"role": "admin"}, headers=headers).status_code == 400 + assert client.get("/admin/organisations/foreign/governance").status_code == 403 + with store.connect() as db: + audit = db.execute("SELECT * FROM tenant_governance_audit").fetchall() + assert len(audit) == 1 + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "false") + assert client.get(url).status_code == 404 and client.put(url, json={}).status_code == 404 + + +def test_auto_eligibility_uses_same_intersection(setup, monkeypatch): + app, store = setup + from services import intelligence_policy + policy(store, models=[]) + monkeypatch.setattr(intelligence_policy, "get_adapter", lambda *args: object()) + candidate = {"model": "openai:gpt", "enabled": True, "entitled": True, "privacy_allowed": True, "billing": "free"} + with app.test_request_context(): + g.authenticated_user = USER + assert not intelligence_policy.eligible(candidate, False, {"API_BASE_URLS": {}}, model_status=lambda _: "enabled") + + +def test_missing_column_no_partial_state_and_repeatable_migration(setup): + app, store = setup + with store.connect() as db: + db.execute("CREATE TABLE old_rows (value TEXT)") + db.execute("INSERT INTO old_rows VALUES ('preserved')") + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + assert db.execute("SELECT value FROM old_rows").fetchone()[0] == "preserved" + db.execute("ALTER TABLE tenant_governance_policies RENAME COLUMN models TO old_models") + with app.test_request_context(): + result = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + assert result.status_code == 503 + with store.connect() as db: + assert db.execute("SELECT COUNT(*) FROM tenant_governance_reservations").fetchone()[0] == 0 + + +def test_key_reject_is_atomic_and_unknown_price_denied(setup): + app, store = setup + policy(store, daily=1_000_000) + with app.test_request_context(): + result = budget_service.BudgetService.check_and_reserve({**USER, "daily_budget_usd": 0.1}, 0.4, NOW) + assert result.error == "budget_exceeded" + assert budget_service.BudgetService.check_and_reserve(USER, None, NOW).error == "unpriced_reservation" + with store.connect() as db: + assert db.execute("SELECT COUNT(*) FROM tenant_governance_components").fetchone()[0] == 0 + assert db.execute("SELECT COUNT(*) FROM tenant_governance_baselines").fetchone()[0] == 0 + + +def test_resolver_cannot_change_principal_and_role_defaults_to_denied(setup): + app, store = setup + governance.register_governance_collaborators(app=app, store=store, tenant_resolver=lambda user: TenantContext("bob", "org1")) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.workspace_context(USER) + assert failure.value.status == 403 + governance.register_governance_collaborators(app=app, store=store, tenant_resolver=lambda user: TENANT) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.workspace_usage(USER) + assert failure.value.status == 403 + + +def test_invalid_flag_warns_once_without_value(monkeypatch, caplog): + governance._warn_once.cache_clear() + monkeypatch.setenv("TENANT_GOVERNANCE_ENABLED", "sensitive-invalid-value") + assert not governance.enabled() and not governance.enabled() + messages = [record.message for record in caplog.records if "TENANT_GOVERNANCE_ENABLED" in record.message] + assert len(messages) == 1 and "sensitive-invalid-value" not in messages[0] + + +def test_private_adapter_preserves_ancestor_denial_level_without_replaying(setup, monkeypatch): + app, _ = setup + calls = [] + monkeypatch.setattr(governance.control_state_d1, "using_d1", lambda: True) + def remote(endpoint, operation, **values): + calls.append((endpoint, operation, values)) + return {"version": 1, "decision": {"allowed": False, "code": "tenant_budget_exceeded", + "status": 429, "level": "team", "period": "daily"}} + monkeypatch.setattr(governance.control_state_d1, "call", remote) + with app.test_request_context(): + with pytest.raises(governance.GovernanceError) as failure: + governance.D1GovernanceStore().call("reserve", id="tg_test") + assert failure.value.code == "tenant_budget_exceeded" and failure.value.level == "team" + assert len(calls) == 1 and calls[0][0] == "tenant_governance" and calls[0][2]["rpc"] is True + + +def test_python_conflicting_settlement_cannot_replace_cost(setup): + app, _ = setup + with app.test_request_context(): + first = budget_service.BudgetService.check_and_reserve(USER, 0.4, NOW) + row = {"selected_model": "openai:gpt", "cost_usd": 0.1, "cost_basis": "usage"} + budget_service.BudgetService.complete(first.reservation, row) + budget_service.BudgetService.complete(first.reservation, row) + with pytest.raises(governance.GovernanceError) as failure: + budget_service.BudgetService.complete(first.reservation, {**row, "cost_usd": 0.2}) + assert failure.value.status == 409 + + +def test_billing_can_read_but_not_write_policy(setup, monkeypatch): + app, store = setup + from routes import usage + monkeypatch.setattr(usage.AuthService, "get_current_user", lambda: {"username": "alice", "is_admin": True}) + governance.register_governance_collaborators(app=app, store=store, + tenant_resolver=lambda user: TENANT, membership_role=lambda principal, org: "billing") + app.config["WTF_CSRF_ENABLED"] = False + usage.register_usage_routes(app, CSRFProtect(app)) + client = app.test_client() + with client.session_transaction() as session: + session["authenticated"] = True + session["user"] = "alice" + url = "/admin/organisations/org1/teams/team1/governance" + assert client.get(url).status_code == 200 + assert client.put(url, json={"daily": 1}, headers={"If-Match": '"0"'}).status_code == 403 + + +class RegisteredGovernanceTest(UnifiedApiTestCase): + def setUp(self): + with patch("config.load_runtime_env", lambda: None): + super().setUp() + import os + os.environ.update({"TENANT_GOVERNANCE_ENABLED": "true", "USAGE_RESERVATIONS_ENABLED": "false", + "USAGE_LEDGER_ENABLED": "false", "MODEL_PRICING_USD_PER_MILLION": '{"opencode:*":{"input":1000,"output":1000}}'}) + self.path = Path(self.temp_dir.name) / "governance.sqlite3" + with sqlite3.connect(self.path) as db: + db.executescript(Path("intelligence-migrations/0034_tenant_governance.sql").read_text()) + self.store = governance.SQLiteGovernanceStore(lambda: sqlite3.connect(self.path)) + governance.register_governance_collaborators(app=self.app, store=self.store, + tenant_resolver=lambda user: TenantContext(user["username"], "org1", "team1"), + membership_role=lambda principal, org: "member") + with patch.object(self.app_module.AuthService, "get_current_user", return_value={"username": "admin", "is_admin": True}): + self.key = self.app_module.AuthService.create_user("alice", scopes=["chat", "models"])["api_key"] + budget_service.BudgetService.reset() + + def tearDown(self): + budget_service.BudgetService.reset() + super().tearDown() + + def chat(self): + response = self._chat_response() + response._content = b'{"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":7}}' + with patch.object(self.app_module.ProxyService, "make_request", return_value=response) as upstream: + result = self.client.post("/v1/chat/completions", headers={"Authorization": f"Bearer {self.key}"}, + json={"model": "opencode:glm-5.2", "messages": [{"role": "user", "content": "test"}], "max_tokens": 5}, buffered=True) + return result, upstream + + def test_missing_table_refuses_real_route_before_upstream(self): + governance.register_governance_collaborators(app=self.app, + store=governance.SQLiteGovernanceStore(lambda: sqlite3.connect(":memory:")), + tenant_resolver=lambda user: TENANT) + response, upstream = self.chat() + assert response.status_code == 503 and response.json["error"]["code"] == "tenant_governance_unavailable" + upstream.assert_not_called() + + def test_workspace_budget_envelope_and_success_attribution(self): + policy(self.store, daily=1_000_000) + response, upstream = self.chat() + assert response.status_code == 200 + upstream.assert_called_once() + with self.store.connect() as db: + row = db.execute("SELECT provider,model,price_basis,charged FROM tenant_governance_reservations").fetchone() + assert row == ("opencode", "opencode:glm-5.2", "usage", 12000) + self.store.call("put", org_id="org1", team_id=None, actor="alice", revision=1, + policy={"daily": 12000}) + denied, upstream = self.chat() + assert denied.status_code == 429 and denied.json["error"]["code"] == "tenant_budget_exceeded" + assert denied.json["error"]["level"] == "organisation" + upstream.assert_not_called() + + def test_usage_route_never_reads_unscoped_history(self): + from routes import usage + with patch.object(usage, "usage_history", side_effect=AssertionError("unscoped history")): + result = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + assert result.status_code == 200 and result.json["workspace"]["org_id"] == "org1" + assert result.json["workspace"]["spent_micro_usd"] == 0 + + def test_disabled_and_legacy_usage_are_identical(self): + import os + os.environ["TENANT_GOVERNANCE_ENABLED"] = "false" + disabled = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + os.environ["TENANT_GOVERNANCE_ENABLED"] = "true" + governance.register_governance_collaborators(app=self.app, store=self.store) + legacy = self.client.get("/v1/usage", headers={"Authorization": f"Bearer {self.key}"}) + assert disabled.status_code == legacy.status_code == 200 and disabled.data == legacy.data diff --git a/tests/test_tenant_governance_worker.mjs b/tests/test_tenant_governance_worker.mjs new file mode 100644 index 00000000..ef78afd5 --- /dev/null +++ b/tests/test_tenant_governance_worker.mjs @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { TenantContext } from "../worker/enterprise-contract.mjs"; +import { handleTenantGovernanceRequest, intersectGrants, createGovernanceLifecycle } from "../worker/tenant-governance-d1.mjs"; + +const NOW = Date.UTC(2026, 9, 9); +const context = { principal_id: "alice", org_id: "org1", team_id: "team1", grants_revision: 0 }; +const migration = readFileSync(new URL("../intelligence-migrations/0034_tenant_governance.sql", import.meta.url), "utf8"); +function database(t, schema = true) { + const sql = new DatabaseSync(":memory:"); + t.after(() => sql.close()); + if (schema) sql.exec(migration); + const db = { prepare(query) { + let values = []; + const statement = { bind(...args) { values = args; return statement; }, + all() { + const prepared = sql.prepare(query); + if (!prepared.columns().length) { + const result = prepared.run(...values); + return { success: true, results: [], meta: { changes: Number(result.changes) } }; + } + return { success: true, results: prepared.all(...values), meta: { changes: 0 } }; + }, first() { return sql.prepare(query).get(...values) ?? null; } }; + return statement; + }, async batch(statements) { + sql.exec("BEGIN IMMEDIATE"); + try { const results = statements.map(s => s.all()); sql.exec("COMMIT"); return results; } + catch (error) { sql.exec("ROLLBACK"); throw error; } + } }; + const env = { TENANT_GOVERNANCE_ENABLED: "true", INTELLIGENCE_DB: db }; + const call = async body => { + const response = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ version: 1, ...body }), + }), env, { now: () => NOW }); + return { status: response.status, ...await response.json() }; + }; + return { sql, db, env, call }; +} +const put = (team_id = null, policy = {}) => ({ operation: "put", org_id: "org1", team_id, + actor: "alice", revision: 0, policy }); +const reserve = (id, changes = {}) => ({ operation: "reserve", id: `tg_${id}`, context, amount: 400_000, + key_daily: 1_000_000, key_monthly: null, base_day: 0, base_month: 0, + day: "2026-10-09", month: "2026-10", ...changes }); +const settle = (id, changes = {}) => ({ operation: "settle", id: `tg_${id}`, cost: 100_000, + provider: "openai", model: "openai:actual", price_basis: "usage", before_dispatch: false, ...changes }); + +test("off, empty and invalid configuration return 404 without storage", async () => { + for (const flag of [undefined, "", "false", "malformed"]) { + const response = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance"), + { TENANT_GOVERNANCE_ENABLED: flag, INTELLIGENCE_DB: { prepare() { assert.fail("disabled storage"); } } }); + assert.equal(response.status, 404); + } +}); + +test("missing schema, missing column and unavailable binding return JSON 503", async t => { + const { call, sql } = database(t, false); + assert.equal((await call(reserve("one"))).error.code, "tenant_governance_unavailable"); + sql.exec(migration); + sql.exec("ALTER TABLE tenant_governance_policies RENAME COLUMN models TO old_models"); + assert.equal((await call(reserve("two"))).status, 503); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_reservations").get().n, 0); +}); + +test("grant intersection never widens key or ancestor grants; absent rows allow", () => { + assert.equal(intersectGrants(true, "openai:gpt", [{ models: ["openai:*"] }, { models: ["openai:gpt"] }]), true); + assert.equal(intersectGrants(true, "openai:other", [{ models: ["openai:gpt"] }]), false); + assert.equal(intersectGrants(false, "openai:gpt", [{ models: ["*"] }]), false); + assert.equal(intersectGrants(true, "search", [{ tools: [] }], "tools"), false); + assert.equal(intersectGrants(true, "search", [] , "tools"), true); +}); + +test("policy CAS and content-free audit reject role escalation", async t => { + const { call, sql } = database(t); + assert.equal((await call(put(null, { daily: 500_000 }))).policy.revision, 1); + assert.equal((await call(put(null, { daily: 600_000 }))).status, 412); + assert.equal((await call({ ...put(null, { role: "admin" }), revision: 1 })).status, 400); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_audit").get().n, 1); + assert.equal((await call({ operation: "policies", context })).policies.length, 1); +}); + +test("every level rejects atomically, settles together and holds unknown costs", async t => { + const { call, sql } = database(t); + await call(put(null, { daily: 1_000_000 })); + await call(put("team1", { daily: 500_000 })); + assert.equal((await call(reserve("one"))).status, 200); + const denied = await call(reserve("two", { amount: 200_000 })); + assert.equal(denied.status, 429); assert.equal(denied.error.level, "team"); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 6); + assert.equal((await call(settle("one"))).status, 200); + assert.equal((await call(reserve("three"))).status, 200); + await call(settle("three", { cost: null, price_basis: null })); + assert.equal((await call(reserve("four", { amount: 1 }))).status, 429); + const usage = await call({ operation: "usage", context, role: "member" }); + assert.equal(usage.spent_micro_usd, 100_000); assert.equal(usage.holds_micro_usd, 400_000); + assert.equal(usage.models.find(row => row.price_basis === "usage").model, "openai:actual"); +}); + +test("concurrent request race admits exactly one; rejection writes nothing", async t => { + const { call, sql } = database(t); + await call(put(null, { monthly: 500_000 })); + const results = await Promise.all(Array.from({ length: 12 }, (_, n) => call(reserve(String(n))))); + assert.equal(results.filter(result => result.status === 200).length, 1); + assert.equal(results.filter(result => result.status === 429).length, 11); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_reservations").get().n, 1); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 2); +}); + +test("key exhaustion denies without touching ancestor or baseline state", async t => { + const { call, sql } = database(t); + const result = await call(reserve("one", { base_day: 800_000 })); + assert.equal(result.status, 429); assert.equal(result.error.code, "budget_exceeded"); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 0); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 0); +}); + +test("private RPC preserves ancestor level and missing-schema remains HTTP 503", async t => { + const { call } = database(t); + await call(put(null, { daily: 100_000 })); + const denied = await call({ ...reserve("one"), rpc: true }); + assert.equal(denied.status, 200); + assert.equal(denied.decision.code, "tenant_budget_exceeded"); + assert.equal(denied.decision.level, "organisation"); + const absent = database(t, false); + assert.equal((await absent.call({ ...reserve("two"), rpc: true })).status, 503); +}); + +test("usage is scoped to caller workspace and own principal unless billing/admin", async t => { + const { call } = database(t); + await call(reserve("one")); await call(settle("one")); + await call(reserve("two", { context: { ...context, principal_id: "bob" } })); await call(settle("two")); + await call(reserve("three", { context: { ...context, org_id: "org2" } })); await call(settle("three")); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + const billing = await call({ operation: "usage", context, role: "billing" }); + assert.equal(billing.spent_micro_usd, 200_000); assert.equal(billing.organisation.spent_micro_usd, 200_000); + assert.equal((await call({ operation: "usage", context, role: "stranger" })).status, 403); +}); + +test("private transport rejects public URLs, content, oversized bodies and invalid money", async t => { + const { call, env } = database(t); + assert.equal((await call({ ...reserve("one"), prompt: "content" })).status, 400); + for (const amount of [-1, 1.5, Number.MAX_SAFE_INTEGER, null]) { + assert.equal((await call(reserve("one", { amount }))).status, 400); + } + const response = await handleTenantGovernanceRequest(new Request("https://public.example/v1/tenant-governance", { method: "POST" }), env); + assert.equal(response.status, 404); + const huge = await handleTenantGovernanceRequest(new Request("http://intelligence.internal/v1/tenant-governance", { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ operation: "put", junk: "x".repeat(33000) }), + }), env); + assert.equal(huge.status, 400); +}); + +test("migration is additive and repeatable with existing rows", t => { + const { sql } = database(t); + sql.exec("CREATE TABLE old_usage (id TEXT); INSERT INTO old_usage VALUES ('kept');"); + sql.exec(migration); + assert.equal(sql.prepare("SELECT id FROM old_usage").get().id, "kept"); +}); + +test("duplicate reservation IDs cannot add components in another scope", async t => { + const { call, sql } = database(t); + assert.equal((await call(reserve("same"))).status, 200); + assert.equal((await call(reserve("same", { context: { ...context, org_id: "org2", principal_id: "bob" } }))).status, 409); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_components").get().n, 6); + assert.equal(sql.prepare("SELECT COUNT(*) n FROM tenant_governance_baselines").get().n, 2); +}); + +test("settlement is idempotent; conflicting amounts cannot replace settled cost", async t => { + const { call } = database(t); + await call(reserve("one")); await call(settle("one")); + assert.equal((await call(settle("one"))).status, 200); + assert.equal((await call(settle("one", { cost: 200_000 }))).status, 409); +}); + +test("native lifecycle uses injected verified tenant, intersection and atomic quotas", async t => { + const { env, call } = database(t); + await call(put(null, { daily: 500_000, models: ["openai:*"] })); + const hooks = createGovernanceLifecycle(env, { call: async body => call(body), + tenant_resolver: async () => new TenantContext(context) }); + await hooks.admit({ user: { id: "alice" }, model: "openai:gpt", key_allowed: true, amount: 400_000, + key_daily: 1_000_000, base_day: 0, base_month: 0 }); + await hooks.before_dispatch(); + await hooks.finalize({ usage: { selected_model: "openai:actual", cost_micro_usd: 100_000, cost_basis: "usage" } }); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + const denied = createGovernanceLifecycle(env, { call: async body => call(body), tenant_resolver: async () => new TenantContext(context) }); + await assert.rejects(denied.admit({ model: "other:gpt", key_allowed: true }), /model_not_allowed/); +}); + +test("disabled and legacy native lifecycle do not call storage", async () => { + for (const flag of ["", "true"]) { + const hooks = createGovernanceLifecycle({ TENANT_GOVERNANCE_ENABLED: flag }, { + call: () => assert.fail("legacy storage"), tenant_resolver: async () => new TenantContext({ principal_id: "alice" }), + }); + await hooks.admit({}); await hooks.before_dispatch(); await hooks.finalize({}); + } +}); + +test("native flat cache events release holds and ancestor errors retain the level", async t => { + const { env, call } = database(t); + await call(put(null, { daily: 500_000 })); + const create = () => createGovernanceLifecycle(env, { call, tenant_resolver: async () => new TenantContext(context) }); + const cached = create(); + await cached.admit({ model: "openai:gpt", key_allowed: true, amount: 400_000 }); + await cached.finalize({ cost_basis: "cache", selected_model: "openai:gpt", handedOff: false }); + assert.equal((await call({ operation: "usage", context, role: "member" })).holds_micro_usd, 0); + const held = create(); + await held.admit({ model: "openai:gpt", key_allowed: true, amount: 400_000 }); + await held.before_dispatch(); + await held.finalize({ cost_basis: "usage", cost_micro_usd: 100_000, selected_model: "openai:actual", handedOff: true }); + assert.equal((await call({ operation: "usage", context, role: "member" })).spent_micro_usd, 100_000); + await assert.rejects(create().admit({ model: "openai:gpt", key_allowed: true, amount: 500_000 }), + error => error.status === 429 && error.details.level === "organisation"); +}); diff --git a/worker/container-env.mjs b/worker/container-env.mjs index e1394018..16038c05 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,7 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "TENANT_GOVERNANCE_ENABLED", "CONTEXT_CANARY_MODE", "CONTEXT_CANARY_POLICY_JSON", "ENTERPRISE_PREVIEW_ENABLED", diff --git a/worker/d1-schema.mjs b/worker/d1-schema.mjs index fb20de73..8a1ab79e 100644 --- a/worker/d1-schema.mjs +++ b/worker/d1-schema.mjs @@ -10,7 +10,7 @@ export const REQUIRED_D1_TABLES = Object.freeze(["intelligence_policy", "intelli "usage_events", "usage_daily", "usage_batches", "media_jobs", "media_job_items", "shadow_eval_samples", "shadow_eval_results", "shadow_eval_config", "shadow_eval_policy_backups", "prompt_templates", - "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts"]); + "config_snapshot_revisions", "config_snapshots", "config_snapshot_applications", "control_revisions", "tool_grants", "generation_cache", "usage_reservation_budgets", "usage_reservations", "usage_reservation_transitions", "session_tiers", "gateway_alert_rules", "gateway_alert_events", "managed_idempotency", "context_pages", "canary_traffic", "semantic_generation_cache", "gateway_batch_files", "gateway_batches", "gateway_batch_items", "hosted_responses", "realtime_tickets", "realtime_sessions", "learned_cooldown", "usage_receipt_keys", "usage_receipt_heads", "usage_receipts", "tenant_governance_policies", "tenant_governance_reservations", "tenant_governance_components", "tenant_governance_baselines", "tenant_governance_audit"]); /** * Readiness for the D1 schema. A deploy that skipped `wrangler d1 migrations apply` reports diff --git a/worker/tenant-governance-d1.mjs b/worker/tenant-governance-d1.mjs new file mode 100644 index 00000000..aef3f34f --- /dev/null +++ b/worker/tenant-governance-d1.mjs @@ -0,0 +1,315 @@ +/** Private governance domain: intersected grants, CAS policy and atomic quota components. */ +import { TenantContext, AuthorityOperation, legacyTenant, MAX_INTEGER } from "./enterprise-contract.mjs"; + +const PERMISSIONS = Object.freeze({ admin: ["edit", "budgets", "usage", "all_usage"], + billing: ["budgets", "usage", "all_usage"], member: ["usage"] }); +const FIELDS = { + get: ["org_id", "team_id"], put: ["org_id", "team_id", "actor", "revision", "policy"], + policies: ["context"], usage: ["context", "role", "since", "until"], + reserve: ["id", "context", "amount", "day", "month", "key_daily", "key_monthly", "base_day", "base_month"], + dispatch: ["id"], settle: ["id", "cost", "provider", "model", "price_basis", "before_dispatch"], +}; +const SCHEMA = [ + "SELECT org_id,team_id,revision,models,tools,daily,monthly FROM tenant_governance_policies LIMIT 0", + "SELECT id,principal_id,org_id,team_id,day,month,estimate,charged,state,revision,provider,model,price_basis,created_at,operation_id FROM tenant_governance_reservations LIMIT 0", + "SELECT reservation_id,level,scope_org,scope_id,period_kind,period,limit_units FROM tenant_governance_components LIMIT 0", + "SELECT principal_id,period,amount FROM tenant_governance_baselines LIMIT 0", + "SELECT operation_id,actor,org_id,team_id,revision,kind,at FROM tenant_governance_audit LIMIT 0", +]; +let warned = false; +export class GovernanceError extends Error { + constructor(code = "tenant_governance_unavailable", status = 503, details = {}) { + super(code); Object.assign(this, { code, status, details }); + } +} +const fail = (code = "invalid_governance_operation", status = 400, details = {}) => { throw new GovernanceError(code, status, details); }; +export function governanceEnabled(env = {}, warn = message => console.warn(message)) { + const raw = env.TENANT_GOVERNANCE_ENABLED; + const flag = raw === undefined ? "" : typeof raw === "string" ? raw.trim().toLowerCase() : "invalid"; + if (["", "false", "0", "off", "no"].includes(flag)) return false; + if (["true", "1", "on", "yes"].includes(flag)) return true; + if (!warned) { warned = true; warn("Invalid TENANT_GOVERNANCE_ENABLED; tenant governance disabled"); } + return false; +} +const permitted = (role, permission) => Object.hasOwn(PERMISSIONS, role) && PERMISSIONS[role].includes(permission); +const integer = n => Number.isSafeInteger(n) && n >= 0 && n < MAX_INTEGER; +function opaque(value) { + if (typeof value !== "string" || !/^[A-Za-z0-9_:.\-]{1,128}$/.test(value)) fail(); +} +function policy(value) { + if (!value || typeof value !== "object" || Array.isArray(value) || + Object.keys(value).some(key => !["models", "tools", "daily", "monthly"].includes(key))) fail(); + const result = { models: value.models ?? null, tools: value.tools ?? null, + daily: value.daily ?? null, monthly: value.monthly ?? null }; + for (const name of ["models", "tools"]) { + if (result[name] !== null && (!Array.isArray(result[name]) || result[name].length > 64 || + result[name].some(pattern => typeof pattern !== "string" || !/^[A-Za-z0-9._:/+@*\-]{1,256}$/.test(pattern)))) fail(); + } + for (const name of ["daily", "monthly"]) if (result[name] !== null && !integer(result[name])) fail(); + return result; +} +function decoded(row) { + try { + return { revision: row.revision, ...policy({ models: row.models === null ? null : JSON.parse(row.models), + tools: row.tools === null ? null : JSON.parse(row.tools), daily: row.daily, monthly: row.monthly }) }; + } catch { throw new GovernanceError(); } +} +export function intersectGrants(keyAllowed, candidate, policies, kind = "models") { + if (!["models", "tools"].includes(kind)) fail(); + return keyAllowed && policies.every(row => row[kind] == null || typeof candidate === "string" && + row[kind].some(pattern => new RegExp(`^${pattern.toLowerCase().split("*") + .map(part => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`).test(candidate.trim().toLowerCase()))); +} +function validate(body) { + if (!body || typeof body !== "object" || Array.isArray(body) || body.version !== 1 || !Object.hasOwn(FIELDS, body.operation)) fail(); + const allowed = new Set(["version", "operation", "rpc", ...FIELDS[body.operation]]); + if (Object.keys(body).some(key => !allowed.has(key))) fail(); + if (body.rpc !== undefined && typeof body.rpc !== "boolean") fail(); + const op = body.operation; + if (["get", "put"].includes(op)) { + opaque(body.org_id); if (body.team_id != null) opaque(body.team_id); + if (op === "put") { opaque(body.actor); if (!integer(body.revision)) fail(); body.policy = policy(body.policy); } + } + if (["policies", "usage", "reserve"].includes(op)) { + try { body.context = new TenantContext(body.context); } catch { fail(); } + if (body.context.org_id === null) fail("tenant_scope_denied", 403); + } + if (op === "usage" && !permitted(body.role, "usage")) fail("tenant_role_denied", 403); + if (op === "usage") for (const key of ["since", "until"]) { + if (body[key] !== undefined && (typeof body[key] !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(body[key]))) fail(); + } + if (["reserve", "dispatch", "settle"].includes(op)) opaque(body.id); + if (op === "reserve") { + for (const key of ["amount", "base_day", "base_month"]) if (!integer(body[key])) fail(); + for (const key of ["key_daily", "key_monthly"]) if (body[key] !== null && !integer(body[key])) fail(); + if (typeof body.day !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(body.day) || + typeof body.month !== "string" || body.month !== body.day.slice(0, 7)) fail(); + } + if (op === "settle") { + if (body.cost !== null && !integer(body.cost)) fail(); + if (body.before_dispatch !== undefined && typeof body.before_dispatch !== "boolean") fail(); + if (body.price_basis != null && !["usage", "estimate", "cache", "released"].includes(body.price_basis)) fail(); + if (body.cost !== null && !["usage", "cache", "released"].includes(body.price_basis)) fail(); + if (body.before_dispatch && (body.cost !== 0 || body.price_basis !== "released")) fail(); + for (const key of ["provider", "model"]) if (body[key] != null && + (typeof body[key] !== "string" || !/^[A-Za-z0-9._:/+@\-]{1,256}$/.test(body[key]))) fail(); + } + return body; +} +async function batch(db, statements) { + const results = await db.batch(statements); + if (!Array.isArray(results) || results.some(result => result.success === false)) throw new GovernanceError(); + return results; +} +async function policies(db, context) { + const rows = await db.prepare("SELECT * FROM tenant_governance_policies WHERE org_id=? AND (team_id='' OR team_id=?)") + .bind(context.org_id, context.team_id ?? "").all(); + return { policies: rows.results.map(decoded) }; +} +async function admin(db, body, now) { + const org = body.org_id, team = body.team_id ?? ""; + if (body.operation === "get") { + const row = await db.prepare("SELECT * FROM tenant_governance_policies WHERE org_id=? AND team_id=?").bind(org, team).first(); + return { policy: row ? decoded(row) : { revision: 0, models: null, tools: null, daily: null, monthly: null } }; + } + const operationId = crypto.randomUUID(); + const { models, tools, daily, monthly } = body.policy; + const results = await batch(db, [ + db.prepare(`INSERT INTO tenant_governance_audit + SELECT ?,?,?,?,?+1,'policy',? WHERE COALESCE((SELECT revision FROM tenant_governance_policies + WHERE org_id=? AND team_id=?),0)=?`).bind(operationId, body.actor, org, team, body.revision, + new Date(now).toISOString(), org, team, body.revision), + db.prepare(`INSERT INTO tenant_governance_policies (org_id,team_id,revision,models,tools,daily,monthly) + SELECT ?,?,?,?,?,?,? WHERE EXISTS (SELECT 1 FROM tenant_governance_audit WHERE operation_id=?) + ON CONFLICT(org_id,team_id) DO UPDATE SET revision=excluded.revision,models=excluded.models, + tools=excluded.tools,daily=excluded.daily,monthly=excluded.monthly`) + .bind(org, team, body.revision + 1, models === null ? null : JSON.stringify(models), + tools === null ? null : JSON.stringify(tools), daily, monthly, operationId), + db.prepare("SELECT p.* FROM tenant_governance_policies p WHERE org_id=? AND team_id=? AND EXISTS (SELECT 1 FROM tenant_governance_audit WHERE operation_id=?)") + .bind(org, team, operationId), + ]); + if (!results[2].results.length) fail("revision_stale", 412); + return { policy: decoded(results[2].results[0]) }; +} + +// All scope reads, admission and component writes run in one D1 batch transaction. +const SCOPES = `WITH scopes(level,scope_org,scope_id,period_kind,period,limit_units) AS ( + SELECT 'key','',?1,'daily',?4,?6 UNION ALL SELECT 'key','',?1,'monthly',?5,?7 + UNION ALL SELECT 'organisation',?2,?2,'daily',?4,(SELECT daily FROM tenant_governance_policies WHERE org_id=?2 AND team_id='') + UNION ALL SELECT 'organisation',?2,?2,'monthly',?5,(SELECT monthly FROM tenant_governance_policies WHERE org_id=?2 AND team_id='') + UNION ALL SELECT 'team',?2,?3,'daily',?4,(SELECT daily FROM tenant_governance_policies WHERE org_id=?2 AND team_id=?3) WHERE ?3!='' + UNION ALL SELECT 'team',?2,?3,'monthly',?5,(SELECT monthly FROM tenant_governance_policies WHERE org_id=?2 AND team_id=?3) WHERE ?3!='' +), totals AS (SELECT s.*, COALESCE((SELECT SUM(CASE WHEN r.state!='settled' THEN r.estimate + WHEN c.period=s.period THEN r.charged ELSE 0 END) FROM tenant_governance_components c + JOIN tenant_governance_reservations r ON r.id=c.reservation_id WHERE c.level=s.level AND c.scope_org=s.scope_org + AND c.scope_id=s.scope_id AND c.period_kind=s.period_kind),0) + + CASE WHEN s.level='key' THEN COALESCE((SELECT amount FROM tenant_governance_baselines WHERE principal_id=?1 AND period=s.period), + CASE WHEN s.period_kind='daily' THEN ?10 ELSE ?11 END) ELSE 0 END AS used FROM scopes s +), denied AS (SELECT * FROM totals WHERE limit_units IS NOT NULL AND (used>=limit_units OR ?8>limit_units-used))`; + +async function reserve(db, body, now) { + const c = body.context; + const args = [c.principal_id, c.org_id, c.team_id ?? "", body.day, body.month, body.key_daily, body.key_monthly, + body.amount, body.id, body.base_day, body.base_month, new Date(now).toISOString(), crypto.randomUUID()]; + const admission = `${SCOPES} INSERT INTO tenant_governance_reservations + (id,principal_id,org_id,team_id,day,month,estimate,state,created_at,operation_id) + SELECT ?9,?1,?2,?3,?4,?5,?8,'reserved',?12,?13 WHERE NOT EXISTS (SELECT 1 FROM denied) + ON CONFLICT(id) DO NOTHING RETURNING id`; + const results = await batch(db, [ + db.prepare(admission).bind(...args), + db.prepare(`${SCOPES} INSERT OR IGNORE INTO tenant_governance_components + SELECT ?9,level,scope_org,scope_id,period_kind,period,limit_units FROM scopes + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?9 AND operation_id=?13)`) + .bind(...args), + db.prepare(`INSERT OR IGNORE INTO tenant_governance_baselines SELECT ?1,?2,?3 + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?4 AND operation_id=?5)`) + .bind(c.principal_id, body.day, body.base_day, body.id, args[12]), + db.prepare(`INSERT OR IGNORE INTO tenant_governance_baselines SELECT ?1,?2,?3 + WHERE EXISTS (SELECT 1 FROM tenant_governance_reservations WHERE id=?4 AND operation_id=?5)`) + .bind(c.principal_id, body.month, body.base_month, body.id, args[12]), + db.prepare(`${SCOPES} SELECT level,period_kind FROM denied ORDER BY CASE level WHEN 'key' THEN 0 WHEN 'organisation' THEN 1 ELSE 2 END LIMIT 1`) + .bind(...args.slice(0, 11)), + db.prepare("SELECT id FROM tenant_governance_reservations WHERE id=?").bind(body.id), + ]); + if (!results[0].results.length) { + if (results[5].results.length) fail("reservation_conflict", 409); + const denied = results[4].results[0]; + if (!denied) fail("reservation_conflict", 409); + fail(denied.level === "key" ? "budget_exceeded" : "tenant_budget_exceeded", 429, + { level: denied.level, period: denied.period_kind }); + } + return { id: body.id }; +} +async function settle(db, body) { + const state = body.operation === "dispatch" ? "dispatched" : body.cost === null ? "unknown" : "settled"; + const row = await db.prepare("SELECT * FROM tenant_governance_reservations WHERE id=?").bind(body.id).first(); + if (!row) fail("reservation_conflict", 409); + if (["settled", "unknown"].includes(row.state)) { + if (row.state === "settled" && body.operation === "settle" && body.cost !== null && row.charged !== body.cost) fail("reservation_conflict", 409); + return { state: row.state }; + } + if (body.before_dispatch && row.state !== "reserved") fail("reservation_conflict", 409); + const results = await batch(db, [db.prepare(`UPDATE tenant_governance_reservations SET state=?,charged=?, + provider=COALESCE(?,provider),model=COALESCE(?,model),price_basis=COALESCE(?,price_basis),revision=revision+1 + WHERE id=? AND revision=? AND state IN ('reserved','dispatched') RETURNING state`) + .bind(state, body.cost ?? null, body.provider ?? null, body.model ?? null, body.price_basis ?? null, body.id, row.revision)]); + if (!results[0].results.length) fail("reservation_conflict", 409); + return { state }; +} +async function usage(db, body) { + const c = body.context, all = permitted(body.role, "all_usage"); + const where = "org_id=? AND (?='' OR team_id=?) AND (?=1 OR principal_id=?) AND day>=? AND day<=?"; + const since = body.since ?? "0000-01-01", until = body.until ?? "9999-12-31"; + const args = [c.org_id, c.team_id ?? "", c.team_id ?? "", all ? 1 : 0, c.principal_id, since, until]; + const results = await batch(db, [ + db.prepare(`SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd, + COUNT(*) AS requests FROM tenant_governance_reservations WHERE ${where}`).bind(...args), + db.prepare(`SELECT provider,model,price_basis,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE ${where} GROUP BY provider,model,price_basis ORDER BY model LIMIT 200`).bind(...args), + ...(all ? [db.prepare(`SELECT COALESCE(SUM(charged),0) AS spent_micro_usd, + COALESCE(SUM(CASE WHEN state!='settled' THEN estimate ELSE 0 END),0) AS holds_micro_usd + FROM tenant_governance_reservations WHERE org_id=? AND day>=? AND day<=?`).bind(c.org_id, since, until)] : []), + db.prepare(`SELECT day,COUNT(*) AS requests,COALESCE(SUM(charged),0) AS spent_micro_usd + FROM tenant_governance_reservations WHERE ${where} GROUP BY day ORDER BY day LIMIT 90`).bind(...args), + ]); + const totals = results[0].results[0]; + return { org_id: c.org_id, team_id: c.team_id, principal_id: c.principal_id, + cost_description: "Gateway cost estimates, not invoices", ...totals, models: results[1].results, + daily: results[all ? 3 : 2].results, range: { since, until }, + ...(all ? { organisation: results[2].results[0], team: c.team_id ? totals : null } : {}) }; +} +async function readBody(request) { + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json" || !request.body) fail(); + const reader = request.body.getReader(), chunks = []; + let length = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + length += value.byteLength; + if (length > 32768) fail(); + chunks.push(value); + } + const bytes = new Uint8Array(length); let offset = 0; + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } + return validate(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))); + } catch (error) { if (error instanceof GovernanceError) throw error; fail(); } + finally { void reader.cancel().catch(() => {}); reader.releaseLock(); } +} +const reply = (value, status = 200) => Response.json({ version: 1, ...value }, + { status, headers: { "cache-control": "no-store", "x-content-type-options": "nosniff" } }); + +/** Mount only in the authenticated private managed-state dispatcher. */ +export async function handleTenantGovernanceRequest(request, env, { now = Date.now } = {}) { + if (!governanceEnabled(env)) return reply({ error: { code: "not_found" } }, 404); + let rpc = false; + try { + const url = new URL(request.url); + if (url.origin !== "http://intelligence.internal" || url.pathname !== "/v1/tenant-governance" || url.search || url.hash) fail("not_found", 404); + if (request.method !== "POST") fail("method_not_allowed", 405); + const body = await readBody(request), db = env.INTELLIGENCE_DB; + rpc = body.rpc === true; + if (!db?.prepare || !db?.batch) throw new GovernanceError(); + for (const query of SCHEMA) await db.prepare(query).all(); + let result; + if (["get", "put"].includes(body.operation)) result = await admin(db, body, now()); + else if (body.operation === "policies") result = await policies(db, body.context); + else if (body.operation === "reserve") result = await reserve(db, body, now()); + else if (body.operation === "usage") result = await usage(db, body); + else result = await settle(db, body); + return reply(result); + } catch (error) { + const failure = error instanceof GovernanceError ? error : new GovernanceError(); + // The bounded Container transport keeps only the code of HTTP errors. + if (rpc && failure.code === "tenant_budget_exceeded") { + return reply({ decision: { allowed: false, code: failure.code, status: failure.status, ...failure.details } }); + } + return reply({ error: { code: failure.code, ...failure.details } }, failure.status); + } +} + +function defaultTenantResolver(event) { + const context = new TenantContext({ principal_id: event.user?.id ?? event.user?.username ?? "legacy" }); + return legacyTenant(new AuthorityOperation({ context, scoped_id: context.principal_id, revision: 0, operation_id: "governance.resolve" })); +} +/** Named native hooks consume trusted admission metadata; no caller header selects tenancy. */ +export function createGovernanceLifecycle(env, { call, tenant_resolver = defaultTenantResolver }) { + let id, finished = false; + const rpc = async (operation, values) => { + const result = await call({ version: 1, operation, ...values }); + const failure = result.error ?? (result.decision?.allowed === false ? result.decision : null); + if (failure) throw new GovernanceError(failure.code, failure.status ?? result.status ?? 503, + { ...(failure.level ? { level: failure.level } : {}), ...(failure.period ? { period: failure.period } : {}) }); + return result; + }; + return { + async admit(event) { + if (!governanceEnabled(env)) return; + const context = await tenant_resolver(event); + if (!(context instanceof TenantContext)) fail("tenant_scope_denied", 403); + if (context.org_id === null) return; + const rows = (await rpc("policies", { context })).policies; + if (!intersectGrants(event.key_allowed === true, event.model, rows)) fail("model_not_allowed", 403); + for (const tool of event.tools ?? []) if (!intersectGrants(event.key_tools?.includes(tool) === true, tool, rows, "tools")) fail("tool_not_allowed", 403); + const candidateId = `tg_${crypto.randomUUID().replaceAll("-", "")}`; + const day = new Date().toISOString().slice(0, 10); + await rpc("reserve", { id: candidateId, context, amount: event.amount, day, month: day.slice(0, 7), + key_daily: event.key_daily ?? null, key_monthly: event.key_monthly ?? null, + base_day: event.base_day ?? 0, base_month: event.base_month ?? 0 }); + id = candidateId; + }, + async before_dispatch() { if (id && !finished) await rpc("dispatch", { id }); }, + async finalize(event) { + if (!id || finished) return; + const row = event.usage ?? event, model = row.selected_model ?? null; + const cached = row.cost_basis === "cache"; + const released = event.handedOff === false; + const measured = !event.cancellationOutcome?.ambiguous && row.cost_basis === "usage"; + await rpc("settle", { id, cost: released || cached ? 0 : measured ? row.cost_micro_usd ?? null : null, + provider: model?.includes(":") ? model.split(":", 1)[0] : null, model, + price_basis: cached ? "cache" : released ? "released" : row.cost_basis ?? null, before_dispatch: released && !cached }); + finished = true; + }, + }; +}