From 26d55eab32089e06eaa30ce7dc28e5bd099fdec7 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 04:24:25 +0530 Subject: [PATCH] feat(gateway): add opt-in output schema validation Validate non-streaming managed results against a caller-supplied JSON schema behind OUTPUT_SCHEMA_VALIDATION_ENABLED (default off). Invalid results return 502 output_schema_violation with bounded, content-free paths; no regeneration or repair. --- docs/output-schema-validation.md | 91 +++++++ routes/tool_repair.py | 53 ++++ services/free_json_contract.py | 26 +- services/output_schema_validation.py | 260 ++++++++++++++++++ tests/test_output_schema_validation.py | 364 +++++++++++++++++++++++++ worker/container-env.mjs | 1 + 6 files changed, 788 insertions(+), 7 deletions(-) create mode 100644 docs/output-schema-validation.md create mode 100644 services/output_schema_validation.py create mode 100644 tests/test_output_schema_validation.py diff --git a/docs/output-schema-validation.md b/docs/output-schema-validation.md new file mode 100644 index 00000000..a49fe214 --- /dev/null +++ b/docs/output-schema-validation.md @@ -0,0 +1,91 @@ +# Explicit output schema validation + +Set `OUTPUT_SCHEMA_VALIDATION_ENABLED=true` to enable the explicit managed request +option `multillm_output_validation`. The default is `false`; an empty setting also +disables validation. Malformed settings disable validation and log one warning +without the setting value. When disabled, the option is neither interpreted nor +removed, and requests and responses retain their existing behavior. + +Send the option on managed Chat Completions, Messages or Responses requests: + +```json +{ + "model": "mimo:mimo-v2.5", + "messages": [{"role": "user", "content": "Return a JSON object with an integer count."}], + "stream": false, + "multillm_output_validation": { + "mode": "strict", + "schema": { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + "additionalProperties": false + } + } +} +``` + +The gateway removes only this option before dispatch. It checks the generated text +as strict JSON against the explicit schema, preserving the original successful +response bytes and headers. For Chat Completions, each choice is checked. For +Messages and Responses, text blocks are joined in their original order. There is +no protocol translation solely for validation. The option does not instruct a +provider to generate JSON; use the provider's supported response format or prompt +alongside it. Existing response format and tool options are preserved. + +With `OUTPUT_SCHEMA_VALIDATION_ENABLED=false`, the same request follows the +existing path, including the option. With validation enabled but the option +absent, the request and response follow the existing path. Raw passthrough and +Worker-native routes keep their existing behavior. Worker requests forwarded to +the managed Flask endpoints use the Flask validator. + +Streaming requests with the option return HTTP 400 +`output_validation_requires_nonstream` before provider dispatch. Malformed options +return HTTP 400 `output_validation_invalid_option`; invalid, oversized or unsupported +schemas return HTTP 400 `output_validation_invalid_schema`. Only `mode: "strict"` +and a schema are accepted in the option. Object and boolean schemas are supported. + +Malformed generated JSON, schema mismatches, oversized responses and exhausted +validation limits return HTTP 502 `output_schema_violation`. For example: + +```json +{ + "error": { + "code": "output_schema_violation", + "message": "Explicit output validation failed", + "reason": "schema_mismatch", + "paths": ["/*"] + } +} +``` + +Error paths retain array indexes and replace all object keys with `*`, so generated +property names cannot disclose response content. Errors omit values, schema text, +validator messages and provider bodies. Reasons include `invalid_json`, +`schema_mismatch`, `invalid_response`, `unexpected_stream`, `body_limit` and +`work_limit`. Upstream HTTP errors retain their existing status, body and headers. +An unexpected successful stream is closed without consuming its content. + +The response envelope is limited to 1 MiB. Schemas are limited to 64 KiB, depth 32 +and 4,096 nodes. Generated JSON is limited to depth 32 and 4,096 nodes. The product +of schema and instance node counts is limited to 100,000. Validation shares a +10,000-keyword budget and 100 ms elapsed budget across choices, with at most 64 +nested keyword evaluations and 100 error paths. Time is checked between keyword +evaluations; it is not a preemptive process deadline. `uniqueItems` supports arrays +of at most 128 items to bound pairwise comparisons. + +Validation uses the installed `jsonschema` package with an empty reference +registry: no network or filesystem schema retrieval is allowed. Local references +are supported; unresolved references fail validation. Draft 4, 6, 7, 2019-09 and +2020-12 are supported, with 2020-12 as the default. Nested dialect and reference +scope changes and regex keywords (`pattern` and `patternProperties`) are rejected because they +cannot reliably observe the work budget. Format annotations do not enable format +assertions. Duplicate object keys, non-finite numbers and surrounding prose are +invalid JSON. This is bounded schema validation, not a universal schema engine +for native Worker traffic. + +Validation never starts a second generation. Tool repair remains separately +opt-in and retains its existing retry policy. A provider may have already charged +for a response that fails validation; a validation error does not grant permission +to retry it. The validator adds no persistent storage or content logs. Existing +retention, request accounting and provider logging policies still apply. diff --git a/routes/tool_repair.py b/routes/tool_repair.py index a8a20ae1..df2cd115 100644 --- a/routes/tool_repair.py +++ b/routes/tool_repair.py @@ -11,6 +11,10 @@ from services.protocol_translation import CHAT, translate_request from services.tool_repair_runtime import HEADER, SKIP_REASK, log_report, repair_completion, repair_mode, summary_header from services.tool_repair_stream import ToolCallBuffer, repair_sse +from services.output_schema_validation import ( + MAX_BODY_BYTES, OutputValidationError, output_contents, prepare_validation, validate_contents, violation, +) +from services.free_json_contract import JsonOutputError, parse_json_output MAX_RESPONSE_BYTES = 8 * 1024 * 1024 @@ -175,3 +179,52 @@ def reask(body): return translate_downstream_response(repaired, source=CHAT, target=source, stream=bool(payload.get("stream")), request_payload=payload) return wrapped + + +def _validate_managed_response(upstream, contract, protocol): + response = as_flask_response(upstream) + if response.status_code >= 400: + return response + try: + if response.mimetype == "text/event-stream": + raise violation("unexpected_stream") + chunks, size = [], 0 + for chunk in response.iter_encoded(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise violation("body_limit") + chunks.append(chunk) + body = b"".join(chunks) + try: + envelope = parse_json_output(body.decode("utf-8")) + except (JsonOutputError, UnicodeError): + raise violation("invalid_response") from None + validate_contents(output_contents(envelope, protocol), contract) + response.set_data(body) + return response + except OutputValidationError: + response.close() + raise + except Exception: + response.close() + raise violation("invalid_response") from None + + +def with_managed_output_validation(dispatch, *, protocol=None): + """Wrap the outer managed dispatcher, before routing and upstream handoff.""" + @wraps(dispatch) + def wrapped(app, auth, metrics, proxy, payload, *args, **kwargs): + try: + prepared, contract = prepare_validation(payload, app.config) + except OutputValidationError as error: + return Response(json.dumps(error.body()), status=error.status, content_type="application/json") + response = dispatch(app, auth, metrics, proxy, prepared, *args, **kwargs) + if contract is None: + return response + endpoint = kwargs.get("endpoint") or (args[0] if args else None) + selected = protocol or ENDPOINT_PROTOCOLS.get(endpoint, "chat") + try: + return _validate_managed_response(response, contract, selected) + except OutputValidationError as error: + return Response(json.dumps(error.body()), status=error.status, content_type="application/json") + return wrapped diff --git a/services/free_json_contract.py b/services/free_json_contract.py index c6b8756d..838a090e 100644 --- a/services/free_json_contract.py +++ b/services/free_json_contract.py @@ -59,17 +59,14 @@ def _check_bounds(value): pending.extend((child, depth + 1) for child in children) -def _check_references(schema): +def iter_schema_nodes(schema): + """Visit schema objects, excluding literal values in enum, const and defaults.""" pending = [schema] while pending: item = pending.pop() if not isinstance(item, dict): continue - for name in ("$ref", "$dynamicRef", "$recursiveRef"): - if name in item and ( - not isinstance(item[name], str) or not item[name].startswith("#") - ): - raise ValueError("Only local schema references are supported") + yield item for key, value in item.items(): if key in _SCHEMA_MAPS and isinstance(value, dict): pending.extend(value.values()) @@ -81,6 +78,15 @@ def _check_references(schema): pending.extend(v for v in value.values() if isinstance(v, dict)) +def _check_references(schema): + for item in iter_schema_nodes(schema): + for name in ("$ref", "$dynamicRef", "$recursiveRef"): + if name in item and ( + not isinstance(item[name], str) or not item[name].startswith("#") + ): + raise ValueError("Only local schema references are supported") + + def _schema_validator(response_format): wrapper = response_format.get("json_schema") if ( @@ -158,7 +164,8 @@ def _finite_float(value): return number -def check_json_output(content, response_format): +def parse_json_output(content): + """Parse strict JSON once, rejecting duplicate keys and non-finite numbers.""" if not isinstance(content, str): raise JsonOutputError("invalid_json") try: @@ -170,6 +177,11 @@ def check_json_output(content, response_format): ) except (ValueError, RecursionError) as error: raise JsonOutputError("invalid_json") from error + return parsed + + +def check_json_output(content, response_format): + parsed = parse_json_output(content) if response_format["type"] == "json_object" and not isinstance(parsed, dict): raise JsonOutputError("schema_mismatch") if response_format["type"] == "json_schema": diff --git a/services/output_schema_validation.py b/services/output_schema_validation.py new file mode 100644 index 00000000..bb358cec --- /dev/null +++ b/services/output_schema_validation.py @@ -0,0 +1,260 @@ +"""Bounded, content-free validation of an explicit managed output contract.""" + +from __future__ import annotations + +import copy +import logging +import os +import time +from dataclasses import dataclass +from typing import Any +from urllib.parse import unquote + +from jsonschema import Draft4Validator, Draft6Validator, Draft7Validator, Draft201909Validator, Draft202012Validator +from jsonschema.validators import extend +from referencing import Registry +from referencing.exceptions import Unresolvable + +from services.free_json_contract import ( + JsonOutputError, iter_schema_nodes, parse_json_output, schema_validator, +) + +OPTION = "multillm_output_validation" +ENV_KEY = "OUTPUT_SCHEMA_VALIDATION_ENABLED" +MAX_BODY_BYTES = 1024 * 1024 +MAX_ERROR_PATHS = 100 +MAX_OUTPUT_DEPTH = 32 +MAX_OUTPUT_NODES = 4096 +MAX_VALIDATION_STEPS = 10000 +MAX_VALIDATION_DEPTH = 64 +MAX_VALIDATION_SECONDS = 0.1 +MAX_NODE_PRODUCT = 100000 +MAX_UNIQUE_ITEMS = 128 +logger = logging.getLogger(__name__) +_warned_invalid_setting = False + + +class OutputValidationError(ValueError): + """An error envelope that cannot contain schema or generated content.""" + + def __init__(self, code, *, status=400, reason=None, paths=()): + self.status = status + self.code = code + self.reason = reason + self.paths = list(paths)[:MAX_ERROR_PATHS] + super().__init__(code) + + def body(self): + error = {"code": self.code, "message": "Explicit output validation failed"} + if self.reason is not None: + error.update(reason=self.reason, paths=self.paths) + return {"error": error} + + +def violation(reason, paths=()): + return OutputValidationError("output_schema_violation", status=502, reason=reason, paths=paths) + + +def enabled(config): + global _warned_invalid_setting + value = config.get(ENV_KEY, os.environ.get(ENV_KEY, "")) + if isinstance(value, bool): + return value + if value is None or value == "": + return False + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"true", "1", "yes", "on"}: + return True + if normalized in {"", "false", "0", "no", "off"}: + return False + if not _warned_invalid_setting: + logger.warning("Invalid OUTPUT_SCHEMA_VALIDATION_ENABLED setting; validation disabled") + _warned_invalid_setting = True + return False + + +def _node_count(value): + pending, count = [(value, 0)], 0 + while pending: + item, depth = pending.pop() + count += 1 + if depth > MAX_OUTPUT_DEPTH or count > MAX_OUTPUT_NODES: + raise ValueError("JSON complexity limit") + children = item.values() if isinstance(item, dict) else item if isinstance(item, list) else () + pending.extend((child, depth + 1) for child in children) + return count + + +@dataclass(frozen=True) +class OutputContract: + schema: Any + validator_class: Any + schema_nodes: int + + +def _pointer_target(schema, reference): + if reference == "#": + return schema + if not reference.startswith("#/"): + return None + target = schema + try: + for part in unquote(reference[2:]).split("/"): + key = part.replace("~1", "/").replace("~0", "~") + target = target[int(key)] if isinstance(target, list) else target[key] + except (KeyError, IndexError, ValueError, TypeError): + return None + return target + + +def _check_work_safety(schema): + pending, visited, scheduled = [schema], set(), {id(schema)} + while pending: + for node in iter_schema_nodes(pending.pop()): + if id(node) in visited: + continue + visited.add(id(node)) + # Regex evaluation cannot be interrupted by a keyword budget. + if "pattern" in node or "patternProperties" in node: + raise ValueError("Regex schemas exceed bounded validation support") + if node is not schema and ("$schema" in node or "$id" in node or "id" in node): + raise ValueError("Nested dialect or reference scope changes are unsupported") + for name in ("$ref", "$dynamicRef", "$recursiveRef"): + if name not in node: + continue + reference = node[name] + if not isinstance(reference, str) or not reference.startswith("#"): + raise ValueError("Only local references are supported") + # JSON pointers may target schemas stored under literal keywords. + target = _pointer_target(schema, reference) + if isinstance(target, dict) and id(target) not in visited and id(target) not in scheduled: + scheduled.add(id(target)) + pending.append(target) + + +def prepare_validation(payload, config): + """Keep the disabled path untouched; prepare and strip only explicit opt-in.""" + if not enabled(config) or OPTION not in payload: + return payload, None + if payload.get("stream"): + raise OutputValidationError("output_validation_requires_nonstream") + option = payload[OPTION] + if (not isinstance(option, dict) or set(option) != {"mode", "schema"} + or option.get("mode") != "strict" or not isinstance(option["schema"], (dict, bool))): + raise OutputValidationError("output_validation_invalid_option") + schema = option["schema"] + try: + validator = schema_validator(schema) + if type(validator) not in {Draft4Validator, Draft6Validator, Draft7Validator, Draft201909Validator, Draft202012Validator}: + raise ValueError("Unsupported bounded validation dialect") + _check_work_safety(schema) + count = _node_count(schema) + except (ValueError, TypeError, RecursionError, OverflowError): + raise OutputValidationError("output_validation_invalid_schema") from None + bounded_schema = copy.deepcopy(schema) + if isinstance(bounded_schema, dict): + # evolve() must retain the budgeted validator when following a root reference. + bounded_schema.pop("$schema", None) + return {key: value for key, value in payload.items() if key != OPTION}, OutputContract( + bounded_schema, type(validator), count, + ) + + +def _bounded_validator(contract): + steps, depth = 0, 0 + expires = time.monotonic() + MAX_VALIDATION_SECONDS + + def bounded(callback, keyword): + def validate(validator, value, instance, schema): + nonlocal steps, depth + steps += 1 + depth += 1 + try: + if (steps > MAX_VALIDATION_STEPS or depth > MAX_VALIDATION_DEPTH + or time.monotonic() >= expires): + raise violation("work_limit") + if keyword == "uniqueItems" and value and isinstance(instance, list) and len(instance) > MAX_UNIQUE_ITEMS: + raise violation("work_limit") + yield from callback(validator, value, instance, schema) + finally: + depth -= 1 + return validate + + cls = extend(contract.validator_class, { + name: bounded(callback, name) for name, callback in contract.validator_class.VALIDATORS.items() + }) + return cls(contract.schema, registry=Registry()) + + +def _redacted_path(path): + # Object keys can be generated content; only structural array indexes survive. + return "".join("/" + (str(part) if isinstance(part, int) else "*") for part in path) + + +def validate_contents(contents, contract): + """All choices share one keyword and elapsed-time budget.""" + validator = _bounded_validator(contract) + nodes = 0 + for content in contents: + nodes += _validate_content(content, contract, validator, nodes) + + +def _validate_content(content, contract, validator, prior_nodes): + try: + instance = parse_json_output(content) + except JsonOutputError: + raise violation("invalid_json") from None + try: + nodes = _node_count(instance) + if (prior_nodes + nodes) * contract.schema_nodes > MAX_NODE_PRODUCT: + raise violation("work_limit") + paths = [] + for error in validator.iter_errors(instance): + paths.append(_redacted_path(error.absolute_path)) + if len(paths) >= MAX_ERROR_PATHS: + break + except OutputValidationError: + raise + except (ValueError, RecursionError, OverflowError): + raise violation("work_limit") from None + except Unresolvable: + raise violation("schema_mismatch") from None + if paths: + raise violation("schema_mismatch", paths) + return nodes + + +def output_contents(envelope, protocol): + """Select structured text without translating or modifying the provider body.""" + if not isinstance(envelope, dict): + raise violation("invalid_response") + if protocol == "chat": + choices = envelope.get("choices") + if not isinstance(choices, list) or not 1 <= len(choices) <= MAX_ERROR_PATHS: + raise violation("invalid_response") + result = [choice.get("message", {}).get("content") for choice in choices + if isinstance(choice, dict) and isinstance(choice.get("message"), dict)] + if len(result) != len(choices): + raise violation("invalid_response") + return result + blocks = envelope.get("content") if protocol == "messages" else None + if protocol == "responses": + output = envelope.get("output") + if not isinstance(output, list): + raise violation("invalid_response") + blocks = [] + for item in output: + if not isinstance(item, dict): + raise violation("invalid_response") + if item.get("type") == "message": + if not isinstance(item.get("content"), list): + raise violation("invalid_response") + blocks.extend(item["content"]) + if not isinstance(blocks, list): + raise violation("invalid_response") + text = [block.get("text") for block in blocks + if isinstance(block, dict) and block.get("type") in {"text", "output_text"}] + if not text or not all(isinstance(part, str) for part in text): + raise violation("invalid_response") + return ["".join(text)] diff --git a/tests/test_output_schema_validation.py b/tests/test_output_schema_validation.py new file mode 100644 index 00000000..f185f870 --- /dev/null +++ b/tests/test_output_schema_validation.py @@ -0,0 +1,364 @@ +"""Explicit managed validation with synthetic dispatch and no provider access.""" + +import json +import importlib +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import Mock, patch + +import pytest +from flask import Flask, Response, request + +from routes import tool_repair +from services import output_schema_validation as validation +from tests.unified_api_test_case import UnifiedApiTestCase + +SCHEMA = {"type": "object", "properties": {"count": {"type": "integer"}}, "required": ["count"]} +OPTION = "multillm_output_validation" + + +@pytest.fixture(autouse=True) +def pin_flag(monkeypatch): + monkeypatch.setenv(validation.ENV_KEY, "false") + + +def completion(content): + return {"choices": [{"message": {"content": content}}]} + + +def payload(schema=SCHEMA, **extra): + return {"model": "synthetic:model", OPTION: {"mode": "strict", "schema": schema}, **extra} + + +def client_for(answer, *, enabled="true", protocol="chat"): + app = Flask(__name__) + app.config.update(TESTING=True, OUTPUT_SCHEMA_VALIDATION_ENABLED=enabled) + dispatch = Mock(return_value=answer) + wrapped = tool_repair.with_managed_output_validation(dispatch, protocol=protocol) + + @app.post("/v1/chat/completions") + def managed(): + return wrapped(app, None, None, None, request.get_json()) + + @app.post("/raw") + def raw(): + return dispatch(app, None, None, None, request.get_json()) + + return app.test_client(), dispatch + + +@pytest.mark.parametrize("enabled", [False, "false", "", "bad-value", None]) +def test_off_preserves_option_and_response_bytes(enabled): + body = b' {"vendor":"native", "choices": []}\n' + answer = Response(body, headers={"X-Synthetic": "keep"}, content_type="application/json") + client, dispatch = client_for(answer, enabled=enabled) + original = payload(schema="not a schema", stream=True) + response = client.post("/v1/chat/completions", json=original) + assert response.data == body and response.headers["X-Synthetic"] == "keep" + assert dispatch.call_args.args[4] == original + assert dispatch.call_count == 1 + + +def test_no_option_and_raw_native_are_unchanged(): + for path, body in [("/v1/chat/completions", {"model": "synthetic:model"}), ("/raw", payload(stream=True))]: + client, dispatch = client_for(Response(b"vendor bytes", content_type="text/plain")) + response = client.post(path, json=body) + assert response.data == b"vendor bytes" + assert dispatch.call_args.args[4] == body + + +def test_valid_response_preserves_bytes_and_strips_only_gateway_option(): + body = json.dumps(completion('{"count":2}'), indent=2).encode() + client, dispatch = client_for(Response(body, content_type="application/json")) + original = payload(response_format={"type": "json_object"}) + response = client.post("/v1/chat/completions", json=original) + assert response.status_code == 200 and response.data == body + assert dispatch.call_args.args[4] == {key: value for key, value in original.items() if key != OPTION} + assert OPTION in original and dispatch.call_count == 1 + + +@pytest.mark.parametrize("content,reason", [ + ("private-response-canary", "invalid_json"), ('{"count":"private-response-canary"}', "schema_mismatch"), + ('{"count":1,"count":2}', "invalid_json"), ("NaN", "invalid_json"), ("1e999", "invalid_json"), +]) +def test_invalid_content_is_redacted_and_never_reasks(content, reason): + client, dispatch = client_for(Response(json.dumps(completion(content)), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload()) + assert response.status_code == 502 + assert response.json["error"]["code"] == "output_schema_violation" + assert response.json["error"]["reason"] == reason + assert "private-response-canary" not in response.get_data(as_text=True) + assert dispatch.call_count == 1 + + +@pytest.mark.parametrize("option", [None, True, {}, {"mode": "repair", "schema": {}}, {"mode": "strict"}, + {"mode": "strict", "schema": []}, {"mode": "strict", "schema": {}, "extra": True}]) +def test_bad_option_fails_before_dispatch(option): + client, dispatch = client_for(Response("unused")) + response = client.post("/v1/chat/completions", json={OPTION: option}) + assert response.status_code == 400 + dispatch.assert_not_called() + + +def test_stream_rejected_before_dispatch(): + client, dispatch = client_for(Response("unused")) + response = client.post("/v1/chat/completions", json=payload(stream=True)) + assert response.status_code == 400 + assert response.json["error"]["code"] == "output_validation_requires_nonstream" + dispatch.assert_not_called() + + +@pytest.mark.parametrize("schema", [ + {"$ref": "https://example.test/private-schema"}, {"$ref": "file:///private-schema"}, + {"$defs": {"hidden": {"$dynamicRef": "https://example.test/schema"}}}, + {"type": "unknown"}, {"description": "x" * 65536}, {"enum": list(range(5000))}, + {"pattern": "(a+)+$"}, {"patternProperties": {"(a+)+$": {}}}, + {"$defs": {"other": {"$schema": "http://json-schema.org/draft-07/schema#"}}}, + {"$schema": "http://json-schema.org/draft-03/schema#"}, + {"$ref": "#/const", "const": {"pattern": "(a+)+$"}}, + {"$ref": "#/const", "const": {"$schema": "http://json-schema.org/draft-07/schema#", "$ref": "#"}}, + {"$defs": {"scope": {"$id": "https://example.test/other", "$ref": "#/const"}}, "const": {"pattern": "(a+)+$"}}, +]) +def test_unsafe_schemas_fail_without_remote_retrieval(schema): + client, dispatch = client_for(Response("unused")) + with patch("urllib.request.urlopen") as fetch: + response = client.post("/v1/chat/completions", json=payload(schema)) + assert response.status_code == 400 + assert "private-schema" not in response.get_data(as_text=True) + dispatch.assert_not_called() + fetch.assert_not_called() + + +def test_deep_schema_is_rejected_before_dispatch(): + schema = {} + for _ in range(40): + schema = {"properties": {"child": schema}} + client, dispatch = client_for(Response("unused")) + assert client.post("/v1/chat/completions", json=payload(schema)).status_code == 400 + dispatch.assert_not_called() + + +@pytest.mark.parametrize("dialect", [None, "http://json-schema.org/draft-07/schema#"]) +def test_local_refs_and_literal_ref_property_work(dialect): + schema = {"$defs": {"number": {"type": "integer"}}, "properties": {"$ref": {"$ref": "#/$defs/number"}}} + if dialect: + schema["$schema"] = dialect + client, dispatch = client_for(Response(json.dumps(completion('{"$ref":2}')), content_type="application/json")) + assert client.post("/v1/chat/completions", json=payload(schema)).status_code == 200 + assert dispatch.call_count == 1 + + +def test_recursive_reference_hits_validation_work_bound(): + client, dispatch = client_for(Response(json.dumps(completion('{}')), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload({"$ref": "#", "$schema": "http://json-schema.org/draft-07/schema#"})) + assert response.status_code == 502 and response.json["error"]["reason"] == "work_limit" + assert dispatch.call_count == 1 + + +def test_work_steps_and_elapsed_limit(monkeypatch): + for budget in ("MAX_VALIDATION_STEPS", "MAX_VALIDATION_SECONDS"): + with monkeypatch.context() as scoped: + scoped.setattr(validation, budget, 0) + client, dispatch = client_for(Response(json.dumps(completion('{"count":2}')), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload()) + assert response.status_code == 502 and response.json["error"]["reason"] == "work_limit" + assert dispatch.call_count == 1 + + +def test_error_paths_are_bounded_and_dynamic_keys_are_redacted(): + content = json.dumps({"private-field-canary": ["private-value-canary"] * 150}) + schema = {"additionalProperties": {"items": {"type": "integer"}}} + client, _ = client_for(Response(json.dumps(completion(content)), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload(schema)) + error = response.json["error"] + assert response.status_code == 502 and len(error["paths"]) == 100 + assert error["paths"][0] == "/*/0" + assert "private-field-canary" not in response.get_data(as_text=True) + assert "private-value-canary" not in response.get_data(as_text=True) + + +@pytest.mark.parametrize("body,reason", [(b"x" * (1024 * 1024 + 1), "body_limit"), (b"not JSON", "invalid_response"), + (b'{"choices":[]}', "invalid_response")]) +def test_invalid_envelope_is_closed(body, reason): + closed = Mock() + answer = Response(iter([body]), content_type="application/json") + answer.call_on_close(closed) + client, dispatch = client_for(answer) + response = client.post("/v1/chat/completions", json=payload()) + assert response.status_code == 502 and response.json["error"]["reason"] == reason + closed.assert_called_once() + assert dispatch.call_count == 1 + + +def test_unexpected_stream_is_closed_without_consuming_it(): + consume = Mock(return_value=b"data: private-response-canary\n\n") + closed = Mock() + answer = Response((consume() for _ in range(1)), content_type="text/event-stream") + answer.call_on_close(closed) + client, _ = client_for(answer) + assert client.post("/v1/chat/completions", json=payload()).status_code == 502 + consume.assert_not_called() + closed.assert_called_once() + + +def test_upstream_error_is_unchanged(): + client, _ = client_for(Response(b"provider error", status=429, headers={"Retry-After": "7"})) + response = client.post("/v1/chat/completions", json=payload()) + assert response.status_code == 429 and response.data == b"provider error" + assert response.headers["Retry-After"] == "7" + + +@pytest.mark.parametrize("protocol,answer", [ + ("messages", {"content": [{"type": "text", "text": '{"count":2}'}]}), + ("responses", {"output": [{"type": "message", "content": [{"type": "output_text", "text": '{"count":2}'}]}]}), +]) +def test_managed_protocols_validate_without_translation(protocol, answer): + body = json.dumps(answer, indent=2).encode() + client, dispatch = client_for(Response(body, content_type="application/json"), protocol=protocol) + response = client.post("/v1/chat/completions", json=payload()) + assert response.status_code == 200 and response.data == body and dispatch.call_count == 1 + + +def test_validation_state_is_request_local(): + client, dispatch = client_for(Response(json.dumps(completion('2')), content_type="application/json")) + assert client.post("/v1/chat/completions", json=payload({"type": "integer"})).status_code == 200 + dispatch.return_value = Response(json.dumps(completion('2')), content_type="application/json") + assert client.post("/v1/chat/completions", json=payload({"type": "string"})).status_code == 502 + assert dispatch.call_count == 2 + + +def test_budget_is_shared_by_all_choices(monkeypatch): + monkeypatch.setattr(validation, "MAX_VALIDATION_STEPS", 1) + body = {"choices": [{"message": {"content": "1"}}, {"message": {"content": "2"}}]} + client, _ = client_for(Response(json.dumps(body), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload({"type": "integer"})) + assert response.status_code == 502 and response.json["error"]["reason"] == "work_limit" + + +@pytest.mark.parametrize("content,schema", [(json.dumps([1] * 5000), True), + (json.dumps([{}] * 129), {"uniqueItems": True}), + ("[" * 40 + "0" + "]" * 40, True)]) +def test_output_complexity_and_quadratic_work_are_bounded(content, schema): + client, _ = client_for(Response(json.dumps(completion(content)), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload(schema)) + assert response.status_code == 502 and response.json["error"]["reason"] == "work_limit" + + +def test_valid_boolean_schema_and_literal_pattern_keys(): + for schema, content in [(True, "null"), ({"const": {"pattern": "literal"}}, '{"pattern":"literal"}')]: + client, _ = client_for(Response(json.dumps(completion(content)), content_type="application/json")) + assert client.post("/v1/chat/completions", json=payload(schema)).status_code == 200 + + +def test_safe_schema_referenced_under_literal_keyword_is_budgeted(): + schema = {"$ref": "#/const", "const": {"type": "integer"}} + client, _ = client_for(Response(json.dumps(completion('1')), content_type="application/json")) + # const remains an assertion in the root schema, so this is a real mismatch. + response = client.post("/v1/chat/completions", json=payload(schema)) + assert response.status_code == 502 and response.json["error"]["reason"] == "schema_mismatch" + + +def test_false_schema_and_missing_reference_are_real_failures(): + for schema in [False, {"$ref": "#/$defs/missing"}]: + client, _ = client_for(Response(json.dumps(completion('{}')), content_type="application/json")) + response = client.post("/v1/chat/completions", json=payload(schema)) + assert response.status_code == 502 and response.json["error"]["reason"] == "schema_mismatch" + + +def test_invalid_setting_logs_once_without_its_value(monkeypatch, caplog): + monkeypatch.setattr(validation, "_warned_invalid_setting", False) + for _ in range(2): + assert not validation.enabled({validation.ENV_KEY: "private-setting-canary"}) + assert sum("validation disabled" in record.message for record in caplog.records) == 1 + assert "private-setting-canary" not in caplog.text + + +def test_env_setting_and_original_schema_are_preserved(monkeypatch): + monkeypatch.setenv(validation.ENV_KEY, "true") + schema = {"$schema": "http://json-schema.org/draft-07/schema#", "type": "integer"} + original = payload(schema) + prepared, contract = validation.prepare_validation(original, {}) + assert OPTION not in prepared and contract is not None + assert original[OPTION]["schema"] == schema and "$schema" in schema + + +def test_validation_does_not_invoke_tool_repair(): + client, dispatch = client_for(Response(json.dumps(completion("invalid")), content_type="application/json")) + with patch.object(tool_repair, "repair_completion") as repair, patch.object(tool_repair, "repair_mode") as mode: + response = client.post("/v1/chat/completions", json=payload(tools=[{"type": "function"}])) + assert response.status_code == 502 and dispatch.call_count == 1 + repair.assert_not_called() + mode.assert_not_called() + + +def test_concurrent_validation_has_independent_budgets(): + def check(number): + _, contract = validation.prepare_validation(payload({"const": number}), {validation.ENV_KEY: True}) + validation.validate_contents([str(number)], contract) + return number + with ThreadPoolExecutor(max_workers=4) as executor: + assert list(executor.map(check, range(16))) == list(range(16)) + + +class RegisteredManagedRouteTests(UnifiedApiTestCase): + """Exercise real route registrations with the explicit integration hook attached.""" + + def setUp(self): + with patch("config.load_runtime_env"): + super().setUp() + self.app.config[validation.ENV_KEY] = True + self.unified = importlib.import_module("routes.unified") + self.repair = importlib.import_module("routes.tool_repair") + for name in ("dispatch_unified_chat_completion", "dispatch_protocol_request"): + original = getattr(self.unified, name) + hook = patch.object(self.unified, name, self.repair.with_managed_output_validation(original)) + hook.start() + self.addCleanup(hook.stop) + + def test_registered_chat_valid_invalid_and_stream(self): + headers = {"Authorization": "Bearer admin-test-key"} + body = payload(messages=[{"role": "user", "content": "synthetic query"}]) + body["model"] = "mimo:mimo-v2.5" + with patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response('{"count":2}')) as send: + response = self.client.post("/v1/chat/completions", json=body, headers=headers) + assert response.status_code == 200 and send.call_count == 1 + assert OPTION not in json.loads(send.call_args.kwargs["data"]) + with patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response("private-response-canary")) as send: + response = self.client.post("/v1/chat/completions", json=body, headers=headers) + assert response.status_code == 502 and send.call_count == 1 + assert "private-response-canary" not in response.get_data(as_text=True) + with patch.object(self.app_module.ProxyService, "make_request") as send: + response = self.client.post("/v1/chat/completions", json={**body, "stream": True}, headers=headers) + assert response.status_code == 400 + send.assert_not_called() + + def test_registered_translated_protocols_and_stream(self): + headers = {"Authorization": "Bearer admin-test-key"} + for path, fields in [("/v1/messages", {"messages": [{"role": "user", "content": "synthetic"}], "max_tokens": 50}), + ("/v1/responses", {"input": "synthetic"})]: + body = payload(**fields) + body["model"] = "mimo:mimo-v2.5" + with patch.object(self.app_module.ProxyService, "make_request", return_value=self._chat_response('{"count":2}')) as send: + response = self.client.post(path, json=body, headers=headers) + assert response.status_code == 200 and send.call_count == 1 + assert OPTION not in json.loads(send.call_args.kwargs["data"]) + with patch.object(self.app_module.ProxyService, "make_request") as send: + response = self.client.post(path, json={**body, "stream": True}, headers=headers) + assert response.status_code == 400 + send.assert_not_called() + + def test_registered_native_protocols_preserve_bytes(self): + headers = {"Authorization": "Bearer admin-test-key"} + for path, model, fields, answer in [ + ("/v1/messages", "opencode:minimax-m3", {"messages": [{"role": "user", "content": "synthetic"}], "max_tokens": 50}, + {"content": [{"type": "text", "text": '{"count":2}'}]}), + ("/v1/responses", "opencode:grok-4.6", {"input": "synthetic"}, + {"output": [{"type": "message", "content": [{"type": "output_text", "text": '{"count":2}'}]}]}), + ]: + upstream = self._chat_response() + upstream._content = json.dumps(answer, indent=2).encode() + body = payload(**fields) + body["model"] = model + with patch.object(self.app_module.ProxyService, "make_request", return_value=upstream) as send: + response = self.client.post(path, json=body, headers=headers) + assert response.status_code == 200 and response.data == upstream.content and send.call_count == 1 + assert OPTION not in json.loads(send.call_args.kwargs["data"]) diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 241cf668..fc7a0a65 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -3,6 +3,7 @@ const DIRECT_ENV_KEYS = [ "SECRET_SCAN_DEFAULT", "CONTENT_RETENTION_ENABLED", "CONTENT_RETENTION_POLICY_JSON", + "OUTPUT_SCHEMA_VALIDATION_ENABLED", "UPSTREAM_DEFAULT_USER_AGENT", "UPSTREAM_DEFAULT_ORIGINATOR", "UPSTREAM_RETRY_AFTER_ADVICE_ENABLED",