From 22988181a4f8367b488507ffac877ee26db24def Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:22:37 +0800 Subject: [PATCH 01/15] =?UTF-8?q?feat(gateway):=20=E6=8C=89=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=A0=81=E5=88=86=E7=B1=BB=20UNREACHED/MAYBE=5FBILLED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../common/src/windup_common/enums/model.py | 4 ++ .../src/windup_framework/gateway/__init__.py | 0 .../src/windup_framework/gateway/classify.py | 43 +++++++++++++++++++ .../src/windup_framework/gateway/types.py | 39 +++++++++++++++++ .../src/windup_framework/providers/sufy.py | 25 +---------- backend/tests/test_gateway_classify.py | 38 ++++++++++++++++ backend/tests/test_sufy_video_download.py | 9 ++-- 7 files changed, 129 insertions(+), 29 deletions(-) create mode 100644 backend/packages/framework/src/windup_framework/gateway/__init__.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/classify.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/types.py create mode 100644 backend/tests/test_gateway_classify.py diff --git a/backend/packages/common/src/windup_common/enums/model.py b/backend/packages/common/src/windup_common/enums/model.py index 0a032bed..816ae8eb 100644 --- a/backend/packages/common/src/windup_common/enums/model.py +++ b/backend/packages/common/src/windup_common/enums/model.py @@ -15,6 +15,9 @@ class ModelErrorType(str, Enum): NETWORK = "network" # 网络错误(连接失败 / DNS),可重试 AUTH = "auth" # 鉴权失败(密钥错 / 失效),不可重试 INVALID_RESPONSE = "invalid_response" # 返回格式错误(如该出图却返回纯文本 / 空) + UNREACHED = "unreached" # 521/522/523/525,请求大概率未到上游 + MAYBE_BILLED = "maybe_billed" # 520/524/其它可能已计费 5xx + UPSTREAM_FAILED = "upstream_failed" # 视频 job failed/cancelled UNKNOWN = "unknown" # 未知错误 @property @@ -24,4 +27,5 @@ def retryable(self) -> bool: ModelErrorType.RATE_LIMIT, ModelErrorType.TIMEOUT, ModelErrorType.NETWORK, + ModelErrorType.UNREACHED, } diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/packages/framework/src/windup_framework/gateway/classify.py b/backend/packages/framework/src/windup_framework/gateway/classify.py new file mode 100644 index 00000000..6423d886 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/classify.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +import math + +from windup_common.enums.model import ModelErrorType + +_MAX_RETRY_WAIT = 30.0 + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def retry_after_seconds(value: str) -> float | None: + try: + delay = float(value) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + delay = (retry_at.astimezone(timezone.utc) - _utc_now()).total_seconds() + if not math.isfinite(delay): + return None + return min(max(delay, 0.0), _MAX_RETRY_WAIT) + + +def classify_http(status: int) -> ModelErrorType: + if status == 429: + return ModelErrorType.RATE_LIMIT + if status in (401, 403): + return ModelErrorType.AUTH + if status in (521, 522, 523, 525): + return ModelErrorType.UNREACHED + if status in (400, 404): + return ModelErrorType.UNKNOWN + if status >= 500: + return ModelErrorType.MAYBE_BILLED + return ModelErrorType.UNKNOWN diff --git a/backend/packages/framework/src/windup_framework/gateway/types.py b/backend/packages/framework/src/windup_framework/gateway/types.py new file mode 100644 index 00000000..24d4f535 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/types.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from windup_common.enums.model import ModelErrorType + + +class Scene(str, Enum): + CHARACTER_IMAGE = "character_image" + CHARACTER_ACTION = "character_action" + + +class Family(str, Enum): + IMAGE_CHAT_DATA_URI = "image.chat_data_uri" + VIDEO_INPUT_REFERENCE = "video.input_reference" + VIDEO_IMAGE_LIST = "video.image_list" + + +class NextStep(str, Enum): + RETRY_SAME = "retry_same" + FALLBACK = "fallback" + FAIL = "fail" + OPEN_AGGREGATOR = "open_aggregator" + + +@dataclass(frozen=True) +class AdapterResult: + ok: bool + body: bytes = b"" + job_id: str | None = None + error_type: ModelErrorType | None = None + http_status: int | None = None + maybe_billed: bool = False + edge_fingerprint: str = "" + output_bytes: int = 0 + expected_bytes: int | None = None + provider_usage: object | None = None + job_status: str | None = None diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index 0733dde8..095430cb 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -23,18 +23,16 @@ from __future__ import annotations import base64 -from datetime import datetime, timezone -from email.utils import parsedate_to_datetime import io import json import logging -import math import re import time import httpx from windup_framework.config.provider import AIProviderSettings, settings +from windup_framework.gateway.classify import _MAX_RETRY_WAIT, retry_after_seconds as _retry_after_seconds from .interfaces import ImageProvider, VideoProvider @@ -273,7 +271,6 @@ def _download(client: httpx.Client, url: str, tries: int = 3) -> bytes: _IMAGE_TRIES = 3 _MIN_IMAGE_BYTES = 5000 _CONNECT_RETRIES = 3 -_MAX_RETRY_WAIT = 30.0 _IMAGE_TIMEOUT_MULTIPLIER = 1.5 # 429 是被限流拒收、必然没计费,所以按次数放开重试。它与 _IMAGE_TRIES 会叠乘,单次 @@ -321,26 +318,6 @@ def _edge_fingerprint(response: httpx.Response) -> str: return " ".join(f"{k}={v}" for k, v in seen.items() if v) or "无可辨识的边缘响应头" -def _utc_now() -> datetime: - return datetime.now(timezone.utc) - - -def _retry_after_seconds(value: str) -> float | None: - try: - delay = float(value) - except ValueError: - try: - retry_at = parsedate_to_datetime(value) - except (TypeError, ValueError, OverflowError): - return None - if retry_at.tzinfo is None: - retry_at = retry_at.replace(tzinfo=timezone.utc) - delay = (retry_at.astimezone(timezone.utc) - _utc_now()).total_seconds() - if not math.isfinite(delay): - return None - return min(max(delay, 0.0), _MAX_RETRY_WAIT) - - def _retry_exhausted_message(status: int, tries: int, fingerprint: str) -> str: """这条文本常常是线上唯一留下的失败记录,少一样就得靠猜是限流、还是哪一跳断的。""" if status == 429: diff --git a/backend/tests/test_gateway_classify.py b/backend/tests/test_gateway_classify.py new file mode 100644 index 00000000..b910c47e --- /dev/null +++ b/backend/tests/test_gateway_classify.py @@ -0,0 +1,38 @@ +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.classify import classify_http, retry_after_seconds + + +def test_522_is_unreached(): + assert classify_http(522) is ModelErrorType.UNREACHED + assert classify_http(525) is ModelErrorType.UNREACHED + assert classify_http(521) is ModelErrorType.UNREACHED + assert classify_http(523) is ModelErrorType.UNREACHED + + +def test_520_and_524_are_maybe_billed(): + assert classify_http(520) is ModelErrorType.MAYBE_BILLED + assert classify_http(524) is ModelErrorType.MAYBE_BILLED + assert classify_http(500) is ModelErrorType.MAYBE_BILLED + + +def test_429_is_rate_limit(): + assert classify_http(429) is ModelErrorType.RATE_LIMIT + + +def test_401_is_auth(): + assert classify_http(401) is ModelErrorType.AUTH + assert classify_http(403) is ModelErrorType.AUTH + + +def test_unreached_is_retryable_maybe_billed_is_not(): + assert ModelErrorType.UNREACHED.retryable + assert ModelErrorType.RATE_LIMIT.retryable + assert not ModelErrorType.MAYBE_BILLED.retryable + assert not ModelErrorType.UPSTREAM_FAILED.retryable + + +def test_retry_after_seconds_number_and_cap(): + assert retry_after_seconds("2") == 2.0 + assert retry_after_seconds("300") == 30.0 + assert retry_after_seconds("invalid") is None + assert retry_after_seconds("NaN") is None diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 8443dab1..3efb1cce 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -15,12 +15,11 @@ import httpx import pytest +from windup_framework.gateway.classify import _utc_now, retry_after_seconds from windup_framework.providers.sufy import ( IncompleteDownloadError, UnsafeDownloadUrlError, _download, - _retry_after_seconds, - _utc_now, ) VIDEO = b"\x00\x01mp4-bytes" * 64 @@ -423,7 +422,7 @@ def h(request): return httpx.Response(200, json=_img_payload(_big_b64())) monkeypatch.setattr( - "windup_framework.providers.sufy._utc_now", + "windup_framework.gateway.classify._utc_now", lambda: datetime(2026, 8, 13, 3, 0, tzinfo=timezone.utc), ) monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) @@ -438,11 +437,11 @@ def test_retry_after_clock_is_utc(): def test_retry_after_accepts_date_without_timezone(monkeypatch): monkeypatch.setattr( - "windup_framework.providers.sufy._utc_now", + "windup_framework.gateway.classify._utc_now", lambda: datetime(2026, 8, 13, 3, 0, tzinfo=timezone.utc), ) - assert _retry_after_seconds("Thu, 13 Aug 2026 03:00:10") == 10.0 + assert retry_after_seconds("Thu, 13 Aug 2026 03:00:10") == 10.0 def test_request_path_comes_from_config_not_a_literal(): From e25fbb87ae40b4957a83bf3956d1dfd694dee744 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:24:39 +0800 Subject: [PATCH 02/15] =?UTF-8?q?feat(gateway):=20=E5=A2=9E=E5=8A=A0=20fal?= =?UTF-8?q?lback=20=E4=B8=8E=E5=8D=95=E4=BB=B7=E9=85=8D=E7=BD=AE=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .env.example | 5 +++++ .../src/windup_framework/config/provider.py | 14 ++++++++++++++ backend/tests/test_sufy_video_download.py | 2 ++ 3 files changed, 21 insertions(+) diff --git a/.env.example b/.env.example index b986b308..93086b4c 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,11 @@ AI_API_KEY=your-ai-api-key # 取值即默认值——不写这两行时跑的就是它们。 AI_IMAGE_MODEL=gemini-2.5-flash-image AI_VIDEO_MODEL=kling-v2-5-turbo +AI_IMAGE_FALLBACKS= +AI_VIDEO_FALLBACKS=kling-v2-6 +AI_IMAGE_UNIT_COST= +AI_VIDEO_UNIT_COST_PER_SECOND= +AI_PRICE_VERSION=2026-08-16 # ── 积分定价 ── QUOTA_REGISTER_GIFT_AMOUNT=100 diff --git a/backend/packages/framework/src/windup_framework/config/provider.py b/backend/packages/framework/src/windup_framework/config/provider.py index faeede24..6d077e14 100644 --- a/backend/packages/framework/src/windup_framework/config/provider.py +++ b/backend/packages/framework/src/windup_framework/config/provider.py @@ -1,5 +1,6 @@ """AI Provider 配置。""" +from pydantic import field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -34,6 +35,19 @@ class AIProviderSettings(BaseSettings): video_model: str = "kling-v2-5-turbo" image_model: str = "gemini-2.5-flash-image" + image_fallbacks: str = "" + video_fallbacks: str = "" + image_unit_cost: float | None = None + video_unit_cost_per_second: float | None = None + price_version: str = "2026-08-16" + + @field_validator("image_unit_cost", "video_unit_cost_per_second", mode="before") + @classmethod + def _empty_cost_is_none(cls, v): + if v == "" or v is None: + return None + return v + @property def normalized_base_url(self) -> str: return self.base_url.rstrip("/") diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 3efb1cce..ea00d066 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -655,6 +655,8 @@ def test_request_shape_is_not_configurable(): for banned in ("image_list_models", "fal_endpoints", "first_frame_field"): assert banned not in fields, f"{banned} 不该进配置，见本用例 docstring" assert {"video_model", "image_model"} <= fields + assert {"image_fallbacks", "video_fallbacks", "image_unit_cost", + "video_unit_cost_per_second", "price_version"} <= fields # ── i2v 主流程（付费路径，此前零覆盖）───────────────────────────────────────── From 28fa46af55cf7aeda071f5e9c2c202c931bcc553 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:30:47 +0800 Subject: [PATCH 03/15] =?UTF-8?q?feat(gateway):=20=E6=8C=89=20family=20?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=20scene=20=E5=9E=8B=E5=8F=B7=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/registry.py | 62 +++++++++++++++++++ backend/tests/test_gateway_registry.py | 29 +++++++++ 2 files changed, 91 insertions(+) create mode 100644 backend/packages/framework/src/windup_framework/gateway/registry.py create mode 100644 backend/tests/test_gateway_registry.py diff --git a/backend/packages/framework/src/windup_framework/gateway/registry.py b/backend/packages/framework/src/windup_framework/gateway/registry.py new file mode 100644 index 00000000..5040637f --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/registry.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.types import Family, Scene + +FAMILIES: dict[str, Family] = { + "gemini-2.5-flash-image": Family.IMAGE_CHAT_DATA_URI, + "kling-v2-5-turbo": Family.VIDEO_INPUT_REFERENCE, + "kling-v2-6": Family.VIDEO_INPUT_REFERENCE, + "kling-video-o1": Family.VIDEO_IMAGE_LIST, # 登记但不允许进 chain +} + + +class RegistryError(ValueError): + pass + + +def _parse_fallbacks(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +class ModelRegistry: + def __init__(self, chains: dict[Scene, tuple[str, ...]]) -> None: + self._chains = chains + + @classmethod + def from_settings(cls, cfg: AIProviderSettings) -> ModelRegistry: + chains = { + Scene.CHARACTER_IMAGE: (cfg.image_model, *_parse_fallbacks(cfg.image_fallbacks)), + Scene.CHARACTER_ACTION: (cfg.video_model, *_parse_fallbacks(cfg.video_fallbacks)), + } + for scene, models in chains.items(): + cls._validate_chain(scene, models) + return cls(chains) + + @staticmethod + def _validate_chain(scene: Scene, models: tuple[str, ...]) -> None: + families: list[Family] = [] + for model in models: + if model not in FAMILIES: + raise RegistryError(f"未登记型号: {model}") + family = FAMILIES[model] + if scene is Scene.CHARACTER_ACTION and family is Family.VIDEO_IMAGE_LIST: + raise RegistryError( + f"family {family.value} 不允许出现在 {scene.value} 链上: {model}" + ) + families.append(family) + if len(set(families)) > 1: + raise RegistryError( + f"scene {scene.value} 链上 family 不一致: {models}" + ) + + def chain(self, scene: Scene) -> tuple[str, ...]: + return self._chains[scene] + + def family_of(self, model: str) -> Family: + if model not in FAMILIES: + raise RegistryError(f"未登记型号: {model}") + return FAMILIES[model] + + def contains(self, scene: Scene, model: str) -> bool: + return model in self._chains[scene] diff --git a/backend/tests/test_gateway_registry.py b/backend/tests/test_gateway_registry.py new file mode 100644 index 00000000..dd8c32e6 --- /dev/null +++ b/backend/tests/test_gateway_registry.py @@ -0,0 +1,29 @@ +import pytest +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.registry import ModelRegistry, RegistryError +from windup_framework.gateway.types import Family, Scene + +def _cfg(**kw) -> AIProviderSettings: + return AIProviderSettings( + image_model="gemini-2.5-flash-image", + video_model="kling-v2-5-turbo", + **kw, + ) + +def test_default_chains(): + r = ModelRegistry.from_settings(_cfg(video_fallbacks="kling-v2-6")) + assert r.chain(Scene.CHARACTER_IMAGE) == ("gemini-2.5-flash-image",) + assert r.chain(Scene.CHARACTER_ACTION) == ("kling-v2-5-turbo", "kling-v2-6") + assert r.family_of("kling-v2-6") is Family.VIDEO_INPUT_REFERENCE + +def test_rejects_image_list_in_video_chain(): + with pytest.raises(RegistryError, match="family"): + ModelRegistry.from_settings(_cfg(video_fallbacks="kling-video-o1")) + +def test_rejects_unknown_model(): + with pytest.raises(RegistryError, match="未登记"): + ModelRegistry.from_settings(_cfg(image_fallbacks="not-a-real-model")) + +def test_empty_fallbacks_ok(): + r = ModelRegistry.from_settings(_cfg(image_fallbacks="", video_fallbacks="")) + assert r.chain(Scene.CHARACTER_ACTION) == ("kling-v2-5-turbo",) From 3a55aa1535281c86a4606742496c0af6365461f3 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:33:23 +0800 Subject: [PATCH 04/15] =?UTF-8?q?feat(gateway):=20522=20=E5=8F=AA=E5=86=8D?= =?UTF-8?q?=E8=AF=95=E4=B8=80=E6=AC=A1=E5=B9=B6=E6=8C=89=E5=85=A5=E5=8F=A3?= =?UTF-8?q?=E7=86=94=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/circuit.py | 28 +++++++++++ .../src/windup_framework/gateway/policy.py | 33 +++++++++++++ backend/tests/test_gateway_policy.py | 49 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 backend/packages/framework/src/windup_framework/gateway/circuit.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/policy.py create mode 100644 backend/tests/test_gateway_policy.py diff --git a/backend/packages/framework/src/windup_framework/gateway/circuit.py b/backend/packages/framework/src/windup_framework/gateway/circuit.py new file mode 100644 index 00000000..1739e0fe --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/circuit.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import time +from collections.abc import Callable + + +class CircuitBreaker: + def __init__( + self, + *, + cooldown_s: float = 60, + monotonic: Callable[[], float] | None = None, + ) -> None: + self._cooldown_s = cooldown_s + self._monotonic = monotonic or time.monotonic + self._open_until: dict[str, float] = {} + + def is_open(self, key: str) -> bool: + until = self._open_until.get(key) + if until is None: + return False + if self._monotonic() >= until: + del self._open_until[key] + return False + return True + + def open(self, key: str) -> None: + self._open_until[key] = self._monotonic() + self._cooldown_s diff --git a/backend/packages/framework/src/windup_framework/gateway/policy.py b/backend/packages/framework/src/windup_framework/gateway/policy.py new file mode 100644 index 00000000..9a98c2d7 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/policy.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.types import NextStep + + +def decide( + *, + error_type: ModelErrorType, + retry_count: int, + has_job_id: bool, +) -> NextStep: + if error_type in (ModelErrorType.MAYBE_BILLED, ModelErrorType.AUTH): + return NextStep.FAIL + if has_job_id and error_type is ModelErrorType.TIMEOUT: + return NextStep.FAIL + if has_job_id and error_type is ModelErrorType.UPSTREAM_FAILED: + return NextStep.FALLBACK + if error_type is ModelErrorType.UNREACHED and retry_count == 0: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.UNREACHED and has_job_id: + return NextStep.FAIL + if error_type is ModelErrorType.UNREACHED: + return NextStep.OPEN_AGGREGATOR + if error_type is ModelErrorType.RATE_LIMIT and retry_count < 2: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.RATE_LIMIT: + return NextStep.FALLBACK + if error_type is ModelErrorType.INVALID_RESPONSE and retry_count < 2: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.INVALID_RESPONSE: + return NextStep.FALLBACK + return NextStep.FAIL diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py new file mode 100644 index 00000000..1ccc5c29 --- /dev/null +++ b/backend/tests/test_gateway_policy.py @@ -0,0 +1,49 @@ +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.policy import decide +from windup_framework.gateway.types import NextStep + + +def test_522_retries_once_then_opens_aggregator(): + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=False) is NextStep.OPEN_AGGREGATOR + + +def test_429_retries_twice_then_fallback(): + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=1, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK + + +def test_520_never_retries(): + assert decide(error_type=ModelErrorType.MAYBE_BILLED, retry_count=0, has_job_id=False) is NextStep.FAIL + + +def test_empty_image_retries_then_fallback(): + assert decide(error_type=ModelErrorType.INVALID_RESPONSE, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.INVALID_RESPONSE, retry_count=2, has_job_id=False) is NextStep.FALLBACK + + +def test_job_id_blocks_fallback_on_unreached(): + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=0, has_job_id=True) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=True) is NextStep.FAIL + + +def test_upstream_job_failure_fallbacks(): + assert decide(error_type=ModelErrorType.UPSTREAM_FAILED, retry_count=0, has_job_id=True) is NextStep.FALLBACK + + +def test_poll_timeout_fails_without_new_job(): + assert decide(error_type=ModelErrorType.TIMEOUT, retry_count=0, has_job_id=True) is NextStep.FAIL + + +def test_circuit_opens_and_cools_down(monkeypatch): + clock = {"t": 0.0} + br = CircuitBreaker(cooldown_s=60, monotonic=lambda: clock["t"]) + assert not br.is_open("aggregator") + br.open("aggregator") + assert br.is_open("aggregator") + clock["t"] = 59.0 + assert br.is_open("aggregator") + clock["t"] = 60.0 + assert not br.is_open("aggregator") From d5d5211f3829be34de2991cd71241f0b5da19f9c Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:39:09 +0800 Subject: [PATCH 05/15] =?UTF-8?q?feat(gateway):=20=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E8=80=97=E6=97=B6=E4=B8=8E=E4=BC=B0=E7=AE=97=E6=88=90=E6=9C=AC?= =?UTF-8?q?=E7=9A=84=20attempt=20trace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/context.py | 42 +++++++++ .../src/windup_framework/gateway/trace.py | 93 +++++++++++++++++++ backend/tests/test_gateway_trace.py | 56 +++++++++++ 3 files changed, 191 insertions(+) create mode 100644 backend/packages/framework/src/windup_framework/gateway/context.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/trace.py create mode 100644 backend/tests/test_gateway_trace.py diff --git a/backend/packages/framework/src/windup_framework/gateway/context.py b/backend/packages/framework/src/windup_framework/gateway/context.py new file mode 100644 index 00000000..ea9c02b1 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/context.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from contextvars import ContextVar, Token +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class CallContext: + request_id: str | None = None + task_id: str | None = None + user_id: str | None = None + start_from_model: str | None = None + + +_call_context: ContextVar[CallContext] = ContextVar("windup_gateway_call_context", default=CallContext()) + + +def current_call_context() -> CallContext: + return _call_context.get() + + +def bind_call_context( + *, + request_id: str | None = None, + task_id: str | None = None, + user_id: str | None = None, + start_from_model: str | None = None, +) -> Callable[[], None]: + token: Token[CallContext] = _call_context.set( + CallContext( + request_id=request_id, + task_id=task_id, + user_id=user_id, + start_from_model=start_from_model, + ) + ) + + def reset() -> None: + _call_context.reset(token) + + return reset diff --git a/backend/packages/framework/src/windup_framework/gateway/trace.py b/backend/packages/framework/src/windup_framework/gateway/trace.py new file mode 100644 index 00000000..c828cc16 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/trace.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import hashlib +import json +import logging +from dataclasses import dataclass, fields +from enum import Enum + +from windup_framework.gateway.types import Scene + +logger = logging.getLogger("windup.gateway") + + +@dataclass +class AttemptTrace: + request_id: str + scene: Scene + model: str + attempt_id: str | None = None + task_id: str | None = None + user_id: str | None = None + family: str | None = None + base_url_host: str | None = None + attempt_index: int | None = None + retry_count: int = 0 + route_reason: str | None = None + circuit_scope: str | None = None + error_type: str | None = None + http_status: int | None = None + edge_fingerprint: str | None = None + job_id: str | None = None + fallback_used: bool = False + outcome: str | None = None + job_status: str | None = None + started_at: str | None = None + ended_at: str | None = None + attempt_latency_ms: int | None = None + total_latency_ms: int | None = None + submit_ms: int | None = None + poll_ms: int | None = None + download_ms: int | None = None + poll_count: int | None = None + retry_after_ms: int | None = None + resend_spent: int | None = None + output_bytes: int | None = None + expected_bytes: int | None = None + input_hash: str | None = None + output_hash: str | None = None + maybe_billed: bool | None = None + cost: float | None = None + price_version: str | None = None + provider_usage: object | None = None + + def as_dict(self) -> dict[str, object]: + out: dict[str, object] = {} + for f in fields(self): + value = getattr(self, f.name) + if isinstance(value, Enum): + value = value.value + out[f.name] = value + return out + + +def estimate_cost( + scene: Scene, + *, + billed: bool, + seconds: int, + image_unit_cost: float | None = None, + video_unit_cost_per_second: float | None = None, +) -> float | None: + if not billed: + return None + if scene == Scene.CHARACTER_IMAGE: + return image_unit_cost + if scene == Scene.CHARACTER_ACTION: + if video_unit_cost_per_second is None: + return None + return video_unit_cost_per_second * seconds + return None + + +def hash_image_input(prompt: str, refs: list[bytes]) -> str: + payload = prompt.encode() + b"\0".join(refs) + return hashlib.sha256(payload).hexdigest() + + +def hash_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def emit(trace: AttemptTrace) -> None: + logger.info("%s", json.dumps(trace.as_dict(), ensure_ascii=False, default=str)) diff --git a/backend/tests/test_gateway_trace.py b/backend/tests/test_gateway_trace.py new file mode 100644 index 00000000..e022b1dc --- /dev/null +++ b/backend/tests/test_gateway_trace.py @@ -0,0 +1,56 @@ +import logging + +from windup_framework.gateway.context import bind_call_context, current_call_context +from windup_framework.gateway.trace import AttemptTrace, emit, estimate_cost +from windup_framework.gateway.types import Scene + +REQUIRED = { + "request_id", "attempt_id", "task_id", "user_id", "scene", "model", "family", + "base_url_host", "attempt_index", "retry_count", "route_reason", "circuit_scope", + "error_type", "http_status", "edge_fingerprint", "job_id", "fallback_used", + "outcome", "job_status", "started_at", "ended_at", "attempt_latency_ms", + "total_latency_ms", "submit_ms", "poll_ms", "download_ms", "poll_count", + "retry_after_ms", "resend_spent", "output_bytes", "expected_bytes", + "input_hash", "output_hash", "maybe_billed", "cost", "price_version", + "provider_usage", +} + + +def test_trace_as_dict_has_required_keys(): + t = AttemptTrace(request_id="r1", scene=Scene.CHARACTER_IMAGE, model="gemini-2.5-flash-image") + keys = set(t.as_dict()) + missing = REQUIRED - keys + assert not missing, missing + + +def test_cost_null_when_unpriced(): + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=True, seconds=5, + image_unit_cost=None, video_unit_cost_per_second=None) is None + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=True, seconds=5, + image_unit_cost=0.02, video_unit_cost_per_second=None) == 0.02 + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=False, seconds=5, + image_unit_cost=0.02, video_unit_cost_per_second=None) is None + assert estimate_cost(Scene.CHARACTER_ACTION, billed=True, seconds=5, + image_unit_cost=None, video_unit_cost_per_second=0.1) == 0.5 + + +def test_cost_never_emits_zero_for_missing_price(): + d = AttemptTrace(request_id="r", scene=Scene.CHARACTER_IMAGE, model="x", cost=None).as_dict() + assert d["cost"] is None + + +def test_context_bind_and_reset(): + assert current_call_context().request_id is None + tok = bind_call_context(request_id="abc", task_id="1", user_id="9", start_from_model="kling-v2-6") + try: + assert current_call_context().request_id == "abc" + assert current_call_context().start_from_model == "kling-v2-6" + finally: + tok() + assert current_call_context().request_id is None + + +def test_emit_logs_json_fields(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + emit(AttemptTrace(request_id="r1", scene=Scene.CHARACTER_IMAGE, model="m")) + assert "r1" in caplog.text From 26992d7369bedf23415da7b47d8fd2c41500dfed Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:48:17 +0800 Subject: [PATCH 06/15] =?UTF-8?q?refactor(providers):=20=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=20adapter=20=E4=B8=80=E6=AC=A1=20POST=20=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=8C=96=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/types.py | 1 + .../src/windup_framework/providers/sufy.py | 195 ++++++--------- backend/tests/test_sufy_video_download.py | 233 +++++++++--------- 3 files changed, 184 insertions(+), 245 deletions(-) diff --git a/backend/packages/framework/src/windup_framework/gateway/types.py b/backend/packages/framework/src/windup_framework/gateway/types.py index 24d4f535..7e984ea6 100644 --- a/backend/packages/framework/src/windup_framework/gateway/types.py +++ b/backend/packages/framework/src/windup_framework/gateway/types.py @@ -37,3 +37,4 @@ class AdapterResult: expected_bytes: int | None = None provider_usage: object | None = None job_status: str | None = None + retry_after_s: float | None = None diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index 095430cb..ea27deb5 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -25,19 +25,18 @@ import base64 import io import json -import logging import re import time import httpx +from windup_common.enums.model import ModelErrorType from windup_framework.config.provider import AIProviderSettings, settings -from windup_framework.gateway.classify import _MAX_RETRY_WAIT, retry_after_seconds as _retry_after_seconds +from windup_framework.gateway.classify import classify_http, retry_after_seconds as _retry_after_seconds +from windup_framework.gateway.types import AdapterResult from .interfaces import ImageProvider, VideoProvider -logger = logging.getLogger("windup.providers.sufy") - # 只有 kling-video-o1 走 image_list;v2 系列 / sora 走 input_reference(字段按模型选,塞错任务会 failed)。 _IMAGE_LIST_MODELS = ("kling-video-o1",) DEFAULT_VIDEO_MODEL = "kling-v2-5-turbo" @@ -265,78 +264,55 @@ def _download(client: httpx.Client, url: str, tries: int = 3) -> bytes: DEFAULT_IMAGE_MODEL = "gemini-2.5-flash-image" -# "调用成功但没返回有效图"的重试次数。与 _download 的网络重试是两码事:那个治连接断, -# 这个治模型返回了一条不含图的正常响应(实测偶发)。也是为什么下面要判 base64 长度 —— -# 返回里可能带一个几十字节的占位串,当图存下去就是一个打不开的文件。 -_IMAGE_TRIES = 3 +# "调用成功但没返回有效图"的下限。返回里可能带一个几十字节的占位串,当图存下去就是一个打不开的文件。 _MIN_IMAGE_BYTES = 5000 _CONNECT_RETRIES = 3 _IMAGE_TIMEOUT_MULTIPLIER = 1.5 -# 429 是被限流拒收、必然没计费,所以按次数放开重试。它与 _IMAGE_TRIES 会叠乘,单次 -# gen_image 的最坏情况因此是:_IMAGE_TRIES × _POST_TRIES = 9 次请求,退避最多睡 -# 6 × _MAX_RETRY_WAIT = 180 秒,加上每次请求自身 timeout × _IMAGE_TIMEOUT_MULTIPLIER。 -_POST_TRIES = 3 - -# 521 源站拒绝连接、523 源站不可达都止步于 TCP 层;522 按 Cloudflare 自己的定义含两种 -# 情形 —— 握手没收到 SYN+ACK,以及连接已建立但源站未及时确认请求,后者请求已经写到源站。 -# 所以"重发不会重复计费"是大概率而非保证,重发次数因此要受 _UNREACHED_RESENDS 约束。 -# -# 判据只看码、不看响应头:``AI_BASE_URL`` 后面挂的是哪家网关不可知,靠 ``cf-ray`` + -# ``server: cloudflare`` 认 Cloudflare 会把真实链路上的 52x 全判否(实测网关自报 -# ``server: APISIX``),整条重试等于不存在。 -# -# 520 与 524 不在此列:连接已建立、请求可能正在源站处理中(524 就是"源站 100 秒没答完"), -# 重发一次就是为同一张图付两次钱。 -_CLOUDFLARE_UNREACHED_STATUS = frozenset({521, 522, 523}) - -# 一次 gen_image 内允许把 52x 重发几次。只按码判就无法排除"网关转发给上游之后才回 52x", -# 与其赌它不存在,不如把最坏情况封成一个小常数:最多多付两张图,且不随上面两层循环叠乘。 -_UNREACHED_RESENDS = 2 - _DIAGNOSTIC_HEADERS = ("server", "cf-ray", "via", "x-served-by", "retry-after") -class _ResendBudget: - """跨 _post 的多次调用共享:叠乘的是循环次数,可重复计费的次数不该跟着叠乘。""" - - def __init__(self) -> None: - self._left = _UNREACHED_RESENDS - self.spent = 0 - - def take(self) -> bool: - if self._left <= 0: - return False - self._left -= 1 - self.spent += 1 - return True - - def _edge_fingerprint(response: httpx.Response) -> str: """52x 出自链路上哪一跳,只能从这几个头看 —— 不记下来,线上就只剩一个状态码可复盘。""" seen = {k: response.headers.get(k) for k in _DIAGNOSTIC_HEADERS} return " ".join(f"{k}={v}" for k, v in seen.items() if v) or "无可辨识的边缘响应头" -def _retry_exhausted_message(status: int, tries: int, fingerprint: str) -> str: - """这条文本常常是线上唯一留下的失败记录,少一样就得靠猜是限流、还是哪一跳断的。""" - if status == 429: - return ( - f"图像服务请求过于频繁(HTTP {status})，连发 {tries} 次均被限流；" - f"请稍后重试或检查服务商额度；{fingerprint}" - ) - return ( - f"图像网关未能连上上游(HTTP {status})，已重发 {tries} 次仍未通；" - f"再重发有重复计费风险，故停止；{fingerprint}" - ) - - # 从响应里捞 data URI。模型把图放在 message.content 里,而不同网关的包裹层级不一样 # (有的 content 是字符串、有的是 parts 数组),故对整个响应 JSON 做一次正则, # 不去猜层级 —— 猜错的代价是"调用成功、费用已产生、但我们说没图"。 _DATA_URI = re.compile(r"data:image/[^;]+;base64,([A-Za-z0-9+/=]{100,})") +def _image_result_from_2xx(resp: httpx.Response) -> AdapterResult: + try: + payload = resp.json() + except ValueError: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应不是 JSON", + ) + found = _DATA_URI.search(json.dumps(payload)) + if not found: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应里没有 data URI", + ) + data = base64.b64decode(found.group(1)) + if len(data) < _MIN_IMAGE_BYTES: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint=f"图只有 {len(data)} 字节(下限 {_MIN_IMAGE_BYTES})", + ) + return AdapterResult(ok=True, body=data, http_status=resp.status_code) + + class SufyImageProvider(ImageProvider): """文生图 / 图生图 provider(OpenAI 兼容的 ``/chat/completions`` 面)。 @@ -368,52 +344,44 @@ def _client(self) -> httpx.Client: transport=httpx.HTTPTransport(retries=_CONNECT_RETRIES), ) - def _post(self, client: httpx.Client, body: dict, resends: _ResendBudget) -> dict: - """发送请求，只重试大概率没被上游收下的失败(429 与 521/522/523)。 - - 为什么把 400 / 404 单独挑出来说:同一把 key 下不同网关的模型目录**不一样**。实测 - ``GET /v1/models``:一个网关 73 个模型、一个图像模型都没有;另一个 134 个、 - 含本模块默认的那个(2026-08-10)。配错 ``AI_BASE_URL`` 时原始报错只是一条 - 404,读的人无从知道该去改配置还是改模型名。 - """ - for attempt in range(1, _POST_TRIES + 1): + def submit_image(self, prompt: str, refs: list[bytes], model: str) -> AdapterResult: + """提示词 + 参考图 → 一次 POST → AdapterResult。重试由 Gateway 做。""" + content: list[dict] = [{"type": "text", "text": prompt}] + for raw in refs: + b64 = base64.b64encode(raw).decode() + content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + }) + body = {"model": model, "messages": [{"role": "user", "content": content}]} + with self._client() as client: resp = client.post(self._cfg.chat_completions_path, json=body) - code = resp.status_code - edge = _edge_fingerprint(resp) - if code in _CLOUDFLARE_UNREACHED_STATUS and not resends.take(): - raise RuntimeError(_retry_exhausted_message(code, resends.spent, edge)) - retryable = code == 429 or code in _CLOUDFLARE_UNREACHED_STATUS - if not retryable: - # 5xx 一律留指纹:要不要人工重发,取决于失败落在链路的哪一跳。 - if code >= 500: - logger.warning( - "图像服务返回 %d,不重发(无法排除请求已到达上游并计费);%s", - code, edge, - ) - break - if attempt == _POST_TRIES: - raise RuntimeError(_retry_exhausted_message(code, _POST_TRIES, edge)) - delay = _retry_after_seconds(resp.headers.get("Retry-After", "")) - if delay is None: - # 上限同样兜住指数退避:上游挂掉时不该把一个图像任务堵成长时间阻塞。 - delay = min(float(2**attempt), _MAX_RETRY_WAIT) - logger.warning( - "图像服务返回 %d，第 %d/%d 次请求，%.2f 秒后重试;%s", - code, - attempt, - _POST_TRIES, - delay, - edge, - ) - time.sleep(delay) + + if 200 <= resp.status_code < 300: + return _image_result_from_2xx(resp) + + error_type = classify_http(resp.status_code) if resp.status_code in (400, 404): - raise RuntimeError( - f"网关 {self._cfg.normalized_base_url} 拒绝了模型 {self._model!r}" + edge = ( + f"网关 {self._cfg.normalized_base_url} 拒绝了模型 {model!r}" f"(HTTP {resp.status_code})。先确认该网关的目录里有它:" f"GET {self._cfg.normalized_base_url}/models —— 不同网关目录不同," f"同一把 key 也是。原始响应:{resp.text[:200]}" ) - return resp.raise_for_status().json() + else: + edge = _edge_fingerprint(resp) + retry_after_header = resp.headers.get("Retry-After") + retry_after_s = ( + _retry_after_seconds(retry_after_header) if retry_after_header else None + ) + return AdapterResult( + ok=False, + error_type=error_type, + http_status=resp.status_code, + maybe_billed=error_type is ModelErrorType.MAYBE_BILLED, + edge_fingerprint=edge, + retry_after_s=retry_after_s, + ) def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: """提示词 + 参考图 → 一张 PNG bytes。拿不到有效图就抛,不返回空 bytes。 @@ -421,28 +389,11 @@ def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: 为什么不返回空 bytes 兜底:上游 ``ImageTaskExecutor`` 会把返回值直接上传对象存储 并写进任务结果,一个 0 字节的"成功"会变成用户看到的一张裂图。 """ - content: list[dict] = [{"type": "text", "text": prompt}] - for raw in refs: - b64 = base64.b64encode(raw).decode() - content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64}"}, - }) - body = {"model": self._model, "messages": [{"role": "user", "content": content}]} - - last = "" - # 预算建在循环外:同一张图的多次尝试共用一份"可能已计费"的额度。 - resends = _ResendBudget() - with self._client() as client: - for attempt in range(1, _IMAGE_TRIES + 1): - payload = self._post(client, body, resends) - found = _DATA_URI.search(json.dumps(payload)) - if found: - data = base64.b64decode(found.group(1)) - if len(data) >= _MIN_IMAGE_BYTES: - return data - last = f"图只有 {len(data)} 字节(下限 {_MIN_IMAGE_BYTES})" - else: - last = "响应里没有 data URI" - logger.warning("文生图第 %d/%d 次没拿到有效图:%s", attempt, _IMAGE_TRIES, last) - raise RuntimeError(f"文生图 {_IMAGE_TRIES} 次均未取得有效图:{last}") + r = self.submit_image(prompt, refs, self._model) + if r.ok: + return r.body + if r.error_type is ModelErrorType.INVALID_RESPONSE: + raise RuntimeError(f"文生图未取得有效图:{r.edge_fingerprint}") + raise RuntimeError( + f"文生图失败(HTTP {r.http_status} {r.error_type}): {r.edge_fingerprint}" + ) diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index ea00d066..957e9fec 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -15,6 +15,7 @@ import httpx import pytest +from windup_common.enums.model import ModelErrorType from windup_framework.gateway.classify import _utc_now, retry_after_seconds from windup_framework.providers.sufy import ( IncompleteDownloadError, @@ -262,6 +263,22 @@ def test_image_provider_extends_request_timeout_by_half(): assert client.timeout.pool == 30 +def test_submit_image_returns_png_on_200(): + r = _image_provider(lambda req: httpx.Response(200, json=_img_payload(_big_b64()))).submit_image("x", [], "gemini-2.5-flash-image") + assert r.ok and r.body.startswith(b"\x89PNG") + + +def test_submit_image_sends_the_model_argument(): + seen: dict = {} + + def h(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_img_payload(_big_b64())) + + _image_provider(h).submit_image("x", [], "gemini-override") + assert seen["body"]["model"] == "gemini-override" + + def test_gen_image_returns_the_decoded_png(): """端点可达而 provider 必抛错 = 每个图像任务稳定 FAILED。实现后必须真能出图。""" def h(request): @@ -290,11 +307,8 @@ def h(request): assert content[1]["image_url"]["url"].startswith("data:image/png;base64,") -def test_response_without_an_image_is_retried_then_raises(): - """模型偶发返回一条不含图的正常响应。重试后仍拿不到必须抛，不能返回空 bytes—— - 上游会把返回值直接上传对象存储并写进任务结果，0 字节的"成功"就是用户看到的裂图。""" - import pytest - +def test_response_without_an_image_is_invalid_response(): + """2xx 但没有图 → INVALID_RESPONSE，一次 POST，不在 adapter 内连打。""" calls = {"n": 0} def h(request): @@ -302,39 +316,46 @@ def h(request): calls["n"] += 1 return httpx.Response(200, json={"choices": [{"message": {"content": "抱歉"}}]}) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert not r.ok + assert r.error_type is ModelErrorType.INVALID_RESPONSE + with pytest.raises(RuntimeError, match="未取得有效图"): _image_provider(h).gen_image("x", []) - assert calls["n"] == 3, "应重试到上限而不是一次就放弃" def test_undersized_image_is_rejected_not_returned(): """响应里可能带一个几十字节的占位串，当图存下去就是打不开的文件。""" import base64 - import pytest - tiny = base64.b64encode(b"\x89PNG" + b"\x00" * 200).decode() + calls = {"n": 0} def h(request): import httpx + calls["n"] += 1 return httpx.Response(200, json=_img_payload(tiny)) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert not r.ok + assert r.error_type is ModelErrorType.INVALID_RESPONSE + with pytest.raises(RuntimeError, match="字节"): _image_provider(h).gen_image("x", []) -def test_first_successful_attempt_stops_retrying(): +def test_first_successful_attempt_is_one_post(): calls = {"n": 0} def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(200, json={"choices": [{"message": {"content": "空"}}]}) return httpx.Response(200, json=_img_payload(_big_b64())) assert _image_provider(h).gen_image("x", []) - assert calls["n"] == 2 + assert calls["n"] == 1 def test_image_client_retries_connection_failures(): @@ -353,27 +374,25 @@ def test_image_client_retries_connection_failures(): client.close() -def test_image_rate_limit_is_retried_after_retry_after(monkeypatch): - """429 表示请求未被网关接收，按 Retry-After 退避后应继续当前图片任务。""" +def test_image_rate_limit_is_retried_after_retry_after(): + """429 → RATE_LIMIT，解析 Retry-After 进 result；adapter 不 sleep（Gateway 才 sleep）。""" calls = {"n": 0} - sleeps: list[float] = [] def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(429, headers={"Retry-After": "0.25"}) - return httpx.Response(200, json=_img_payload(_big_b64())) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, headers={"Retry-After": "0.25"}) - assert _image_provider(h).gen_image("x", []) - assert calls["n"] == 2 - assert sleeps == [0.25] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == 0.25 + assert not r.ok + assert r.maybe_billed is False -def test_image_rate_limit_exhaustion_has_actionable_error(monkeypatch): - """持续 429 不能泄漏 httpx 异常，也不能无限重试。""" +def test_image_rate_limit_exhaustion_has_actionable_error(): + """持续 429 不能泄漏 httpx 异常；adapter 一次一枪，重试留给 Gateway。""" calls = {"n": 0} def h(request): @@ -381,54 +400,44 @@ def h(request): calls["n"] += 1 return httpx.Response(429, text='{"error":{"message":"quota exceeded"}}') - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match="稍后重试或检查服务商额度"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == 3 + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert not r.ok @pytest.mark.parametrize( - ("retry_after", "expected"), [("invalid", 2.0), ("NaN", 2.0), ("300", 30.0)] + ("retry_after", "expected"), [("invalid", None), ("NaN", None), ("300", 30.0)] ) -def test_image_rate_limit_wait_has_fallback_and_cap(monkeypatch, retry_after, expected): - calls = {"n": 0} - sleeps: list[float] = [] - +def test_image_rate_limit_wait_has_fallback_and_cap(retry_after, expected): def h(request): import httpx - calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(429, headers={"Retry-After": retry_after}) - return httpx.Response(200, json=_img_payload(_big_b64())) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, headers={"Retry-After": retry_after}) - assert _image_provider(h).gen_image("x", []) - assert sleeps == [expected] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == expected def test_image_rate_limit_accepts_http_date(monkeypatch): calls = {"n": 0} - sleeps: list[float] = [] def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response( - 429, headers={"Retry-After": "Thu, 13 Aug 2026 03:00:10 GMT"} - ) - return httpx.Response(200, json=_img_payload(_big_b64())) + return httpx.Response( + 429, headers={"Retry-After": "Thu, 13 Aug 2026 03:00:10 GMT"} + ) monkeypatch.setattr( "windup_framework.gateway.classify._utc_now", lambda: datetime(2026, 8, 13, 3, 0, tzinfo=timezone.utc), ) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) - assert _image_provider(h).gen_image("x", []) - assert sleeps == [10.0] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == 10.0 def test_retry_after_clock_is_utc(): @@ -467,21 +476,22 @@ def h(request): pytest.param({"cf-ray": "8f2b1c4d5e6a7890-SJC", "server": "nginx"}, id="relayed-cf-ray"), ]) @pytest.mark.parametrize("code", [521, 522, 523]) -def test_52x_is_retried_whatever_the_edge_looks_like(monkeypatch, code, headers): +def test_52x_is_classified_unreached_in_one_call(code, headers): """判据只看码:``AI_BASE_URL`` 后面挂哪家网关不可知,靠响应头认 Cloudflare 会把真实 - 链路上的 52x 全判否(实测网关自报 ``server: APISIX``),整条重试等于不存在。 + 链路上的 52x 全判否(实测网关自报 ``server: APISIX``)。adapter 一次 POST 分类即可。 """ calls = {"n": 0} def h(request): calls["n"] += 1 - if calls["n"] <= 2: - return httpx.Response(code, headers=headers, text="Connection timed out") - return httpx.Response(200, json=_img_payload(_big_b64())) + return httpx.Response(code, headers=headers, text="Connection timed out") - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - assert _image_provider(h).gen_image("x", []).startswith(b"\x89PNG") - assert calls["n"] == 3, "必须真的重发，而不是靠外层出图循环碰运气" + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.UNREACHED + assert not r.ok + assert r.http_status == code + assert r.maybe_billed is False @pytest.mark.parametrize("code", [520, 524]) @@ -493,103 +503,74 @@ def h(request): seen["n"] += 1 return httpx.Response(code, headers={"server": "cloudflare"}, text="ambiguous") - with pytest.raises(httpx.HTTPStatusError): - _image_provider(h).gen_image("x", []) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") assert seen["n"] == 1, f"HTTP {code} 被重试了,会重复计费" + assert r.error_type is ModelErrorType.MAYBE_BILLED + assert r.maybe_billed is True + assert not r.ok -def test_retryable_set_excludes_the_codes_that_may_have_billed(): - """常量本身也钉一道:改集合的人不必先读懂 _post 才发现自己开了重复计费的洞。""" - from windup_framework.providers.sufy import _CLOUDFLARE_UNREACHED_STATUS - - assert _CLOUDFLARE_UNREACHED_STATUS == {521, 522, 523} - - -def test_unreached_resends_are_capped_across_the_whole_gen_image(monkeypatch): - """52x 的"没到上游"是大概率不是保证(CF 的 522 含"连上了但源站没及时确认"), - 所以可重复计费的重发次数按整次 gen_image 封顶,不跟着内外两层循环叠乘。 - """ - from windup_framework.providers.sufy import _UNREACHED_RESENDS - +def test_unreached_resends_are_capped_across_the_whole_gen_image(): + """adapter 不再连打 52x；一次 POST + UNREACHED，重发次数由 Gateway 封顶。""" calls = {"n": 0} def h(request): calls["n"] += 1 - if calls["n"] % 2: - return httpx.Response(522, headers={"server": "APISIX"}, text="timed out") - return httpx.Response(200, json={"choices": [{"message": {"content": "无图"}}]}) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + return httpx.Response(522, headers={"server": "APISIX"}, text="timed out") - with pytest.raises(RuntimeError, match=r"已重发 2 次"): - _image_provider(h).gen_image("x", []) - # 预算若按 _post 调用各算一份,外层三轮就会重发 3 次而不是 2 次。 - assert calls["n"] == 2 * _UNREACHED_RESENDS + 1 + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.UNREACHED -def test_exhausted_retries_report_the_edge_fingerprint(monkeypatch): - """三次全 52x 正是最需要复盘的场景,而它唯一留下的就是这条异常文本。""" +def test_exhausted_retries_report_the_edge_fingerprint(): + """52x 复盘靠边缘指纹，不再依赖「已重发 N 次」异常文本。""" def h(request): return httpx.Response(522, headers={"server": "APISIX", "cf-ray": "8f2b-SJC"}) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match=r"522.*已重发 2 次.*server=APISIX"): - _image_provider(h).gen_image("x", []) - + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNREACHED + assert "server=APISIX" in r.edge_fingerprint + assert "cf-ray=8f2b-SJC" in r.edge_fingerprint -def test_rate_limit_exhaustion_also_reports_the_fingerprint(monkeypatch): - """限流与"网关连不上上游"要能一眼分开 —— 两者的处置完全不同。""" - from windup_framework.providers.sufy import _POST_TRIES +def test_rate_limit_exhaustion_also_reports_the_fingerprint(): + """限流与「网关连不上上游」要能一眼分开。""" calls = {"n": 0} def h(request): calls["n"] += 1 return httpx.Response(429, headers={"server": "APISIX"}, text="slow down") - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match=r"过于频繁.*连发 3 次.*server=APISIX"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == _POST_TRIES - + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert "server=APISIX" in r.edge_fingerprint -def test_unreached_backoff_is_capped(monkeypatch): - """上游挂掉时不该把一个图像任务堵成长时间阻塞。""" - from windup_framework.providers.sufy import _MAX_RETRY_WAIT - sleeps: list[float] = [] +def test_unreached_backoff_is_capped(): + """Retry-After 过大时解析结果仍封顶，adapter 不 sleep。""" + from windup_framework.gateway.classify import _MAX_RETRY_WAIT def h(request): return httpx.Response(522, headers={**_CF_EDGE, "Retry-After": "9999"}) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) - - with pytest.raises(RuntimeError, match="522"): - _image_provider(h).gen_image("x", []) - assert sleeps and max(sleeps) <= _MAX_RETRY_WAIT - + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNREACHED + assert r.retry_after_s == _MAX_RETRY_WAIT -def test_worst_case_request_count_and_wait_are_bounded(monkeypatch): - """内外两层重试会叠乘,最坏情况必须是个说得出的数,而不是"看情况"。""" - from windup_framework.providers.sufy import _IMAGE_TRIES, _MAX_RETRY_WAIT, _POST_TRIES +def test_worst_case_request_count_and_wait_are_bounded(): + """adapter 一次一枪，最坏情况就是 1 次 POST。""" calls = {"n": 0} - sleeps: list[float] = [] def h(request): calls["n"] += 1 - if calls["n"] % _POST_TRIES: - return httpx.Response(429, text="slow down") - return httpx.Response(200, json={"choices": [{"message": {"content": "无图"}}]}) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, text="slow down") - with pytest.raises(RuntimeError, match="均未取得有效图"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == _IMAGE_TRIES * _POST_TRIES == 9 - assert sum(sleeps) <= _IMAGE_TRIES * (_POST_TRIES - 1) * _MAX_RETRY_WAIT + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT @pytest.mark.parametrize("code", [400, 404]) @@ -601,6 +582,12 @@ def h(request): import httpx return httpx.Response(code, text='{"error":{"message":"model not found"}}') + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNKNOWN + assert "/models" in r.edge_fingerprint + assert r.http_status == code + assert not r.ok + with pytest.raises(RuntimeError, match=r"/models"): _image_provider(h).gen_image("x", []) From 23fb730789053d255a1dbcf2b52a9d98f95574e7 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:59:55 +0800 Subject: [PATCH 07/15] =?UTF-8?q?feat(gateway):=20=E5=9B=BE=E5=83=8F?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=8C=89=E7=AD=96=E7=95=A5=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E4=B8=8E=20Fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/__init__.py | 3 + .../src/windup_framework/gateway/image.py | 322 ++++++++++++++++++ .../src/windup_framework/gateway/registry.py | 1 + backend/tests/test_gateway_image.py | 101 ++++++ backend/tests/test_gateway_registry.py | 9 + 5 files changed, 436 insertions(+) create mode 100644 backend/packages/framework/src/windup_framework/gateway/image.py create mode 100644 backend/tests/test_gateway_image.py diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py index e69de29b..084564d7 100644 --- a/backend/packages/framework/src/windup_framework/gateway/__init__.py +++ b/backend/packages/framework/src/windup_framework/gateway/__init__.py @@ -0,0 +1,3 @@ +from windup_framework.gateway.image import ImageGateway, build_image_gateway + +__all__ = ["ImageGateway", "build_image_gateway"] diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py new file mode 100644 index 00000000..e17e2b3c --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import time +import uuid +from datetime import datetime, timezone +from urllib.parse import urlparse + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.policy import decide +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.trace import ( + AttemptTrace, + emit, + estimate_cost, + hash_bytes, + hash_image_input, +) +from windup_framework.gateway.types import NextStep, Scene + +_CIRCUIT = CircuitBreaker() +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class ImageGateway: + def __init__(self, registry, adapter, circuit, settings) -> None: + self._registry = registry + self._adapter = adapter + self._circuit = circuit + self._settings = settings + + def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = hash_image_input(prompt, refs) + host = urlparse(self._settings.base_url).hostname + last_http_status: int | None = None + fallback_used = False + fallback_reason: str | None = None + + chain = list(self._registry.chain(Scene.CHARACTER_IMAGE)) + if ctx.start_from_model and ctx.start_from_model in chain: + start_i = chain.index(ctx.start_from_model) + models = chain[start_i:] + else: + start_i = 0 + models = chain + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + raise RuntimeError( + f"image gateway failed request_id={request_id} http_status={http_status}" + ) + + if self._circuit.is_open("aggregator"): + model = models[0] if models else "" + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="aggregator", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=False, + ) + fail(None) + + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="model", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + ) + continue + + if i == 0: + route_reason = ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" + ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + result = self._adapter.submit_image(prompt, refs, model) + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + billed = result.ok or result.maybe_billed + cost = estimate_cost( + Scene.CHARACTER_IMAGE, + billed=billed, + seconds=0, + image_unit_cost=self._settings.image_unit_cost, + video_unit_cost_per_second=self._settings.video_unit_cost_per_second, + ) + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + if result.ok: + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=None, + outcome="fallback_success" if fallback_used else "success", + input_hash=input_hash, + output_hash=hash_bytes(result.body), + total_latency_ms=total_ms(), + fallback_used=fallback_used, + http_status=result.http_status, + maybe_billed=True, + cost=cost, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + output_bytes=len(result.body), + expected_bytes=result.expected_bytes, + provider_usage=result.provider_usage, + edge_fingerprint=result.edge_fingerprint or None, + job_id=result.job_id, + job_status=result.job_status, + retry_after_ms=retry_after_ms, + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=bool(result.job_id), + ) + circuit_scope = None + if step is NextStep.OPEN_AGGREGATOR: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=circuit_scope, + outcome="failed", + input_hash=input_hash, + output_hash=None, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + http_status=result.http_status, + error_type=error_type.value, + maybe_billed=result.maybe_billed, + cost=cost, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + output_bytes=result.output_bytes or None, + expected_bytes=result.expected_bytes, + provider_usage=result.provider_usage, + edge_fingerprint=result.edge_fingerprint or None, + job_id=result.job_id, + job_status=result.job_status, + retry_after_ms=retry_after_ms, + ) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + break + fail(last_http_status) + + fail(last_http_status) + + def _emit( + self, + *, + request_id: str, + ctx, + model: str, + host: str | None, + attempt_index: int, + retry_count: int, + route_reason: str, + circuit_scope: str | None, + outcome: str, + input_hash: str, + total_latency_ms: int, + fallback_used: bool, + output_hash: str | None = None, + http_status: int | None = None, + error_type: str | None = None, + maybe_billed: bool | None = None, + cost: float | None = None, + started_at: str | None = None, + ended_at: str | None = None, + attempt_latency_ms: int | None = None, + resend_spent: int | None = 0, + output_bytes: int | None = None, + expected_bytes: int | None = None, + provider_usage: object | None = None, + edge_fingerprint: str | None = None, + job_id: str | None = None, + job_status: str | None = None, + retry_after_ms: int | None = None, + ) -> None: + family = None + if model: + family = self._registry.family_of(model).value + emit( + AttemptTrace( + request_id=request_id, + attempt_id=str(uuid.uuid4()), + task_id=ctx.task_id, + user_id=ctx.user_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + family=family, + base_url_host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=circuit_scope, + error_type=error_type, + http_status=http_status, + edge_fingerprint=edge_fingerprint, + job_id=job_id, + fallback_used=fallback_used, + outcome=outcome, + job_status=job_status, + started_at=started_at or _utc_now(), + ended_at=ended_at or _utc_now(), + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_latency_ms, + submit_ms=None, + poll_ms=None, + download_ms=None, + poll_count=None, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + output_bytes=output_bytes, + expected_bytes=expected_bytes, + input_hash=input_hash, + output_hash=output_hash, + maybe_billed=maybe_billed, + cost=cost, + price_version=self._settings.price_version, + provider_usage=provider_usage, + ) + ) + + +def build_image_gateway(config=None, *, adapter=None, circuit=None) -> ImageGateway: + cfg: AIProviderSettings = config or default_settings + if adapter is None: + from windup_framework.providers.sufy import SufyImageProvider + + adapter = SufyImageProvider(config=cfg) + return ImageGateway( + ModelRegistry.from_settings(cfg), + adapter, + circuit if circuit is not None else _CIRCUIT, + cfg, + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/registry.py b/backend/packages/framework/src/windup_framework/gateway/registry.py index 5040637f..f6ac2564 100644 --- a/backend/packages/framework/src/windup_framework/gateway/registry.py +++ b/backend/packages/framework/src/windup_framework/gateway/registry.py @@ -5,6 +5,7 @@ FAMILIES: dict[str, Family] = { "gemini-2.5-flash-image": Family.IMAGE_CHAT_DATA_URI, + "gemini-2.5-flash-image-alt": Family.IMAGE_CHAT_DATA_URI, # test double; not a production default "kling-v2-5-turbo": Family.VIDEO_INPUT_REFERENCE, "kling-v2-6": Family.VIDEO_INPUT_REFERENCE, "kling-video-o1": Family.VIDEO_IMAGE_LIST, # 登记但不允许进 chain diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py new file mode 100644 index 00000000..b7c83e60 --- /dev/null +++ b/backend/tests/test_gateway_image.py @@ -0,0 +1,101 @@ +import logging + +import pytest +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.image import ImageGateway +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import AdapterResult + +UNREACHED = AdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +BILLED = AdapterResult(ok=False, error_type=ModelErrorType.MAYBE_BILLED, http_status=520) +PNG = AdapterResult(ok=True, body=b"\x89PNG\r\n" + b"x" * 5000) + + +class FakeImageAdapter: + def __init__(self, by_model: dict[str, list[AdapterResult]]): + self.by_model = {k: list(v) for k, v in by_model.items()} + self.calls: list[str] = [] + + def submit_image(self, prompt, refs, model): + self.calls.append(model) + q = self.by_model[model] + return q.pop(0) if q else AdapterResult(ok=False, error_type=ModelErrorType.UNKNOWN) + + +def _make_gw(adapter, **kw): + circuit = kw.pop("circuit", None) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks=kw.pop("image_fallbacks", ""), + **kw, + ) + registry = ModelRegistry.from_settings(cfg) + return ImageGateway(registry, adapter, circuit or CircuitBreaker(), cfg) + + +def test_522_retries_same_model_once_and_does_not_fallback(): + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + with pytest.raises(RuntimeError, match="522"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + + +def test_aggregator_circuit_skips_fallback_model(): + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + br = CircuitBreaker(cooldown_s=60) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt", circuit=br) + with pytest.raises(RuntimeError): + gw.gen_image("p", []) + assert "gemini-2.5-flash-image-alt" not in ad.calls + assert br.is_open("aggregator") + + +def test_429_falls_back_to_next_model(monkeypatch): + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert ad.calls[-1] == "gemini-2.5-flash-image-alt" + assert ad.calls.count("gemini-2.5-flash-image") == 3 + + +def test_520_does_not_retry(): + ad = FakeImageAdapter({"gemini-2.5-flash-image": [BILLED, PNG]}) + gw = _make_gw(ad, image_fallbacks="") + with pytest.raises(RuntimeError, match="520"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image"] + + +def test_empty_image_then_fallback(): + empty = AdapterResult(ok=False, error_type=ModelErrorType.INVALID_RESPONSE) + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [empty, empty, empty], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + gw.gen_image("p", []) + assert ad.calls.count("gemini-2.5-flash-image") == 3 + assert ad.calls[-1] == "gemini-2.5-flash-image-alt" + + +def test_success_trace_has_latency_and_null_cost_by_default(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + ad = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + gw = _make_gw(ad, image_fallbacks="") + gw.gen_image("p", []) + assert "total_latency_ms" in caplog.text + assert '"cost": null' in caplog.text or '"cost":null' in caplog.text diff --git a/backend/tests/test_gateway_registry.py b/backend/tests/test_gateway_registry.py index dd8c32e6..a0da0f39 100644 --- a/backend/tests/test_gateway_registry.py +++ b/backend/tests/test_gateway_registry.py @@ -27,3 +27,12 @@ def test_rejects_unknown_model(): def test_empty_fallbacks_ok(): r = ModelRegistry.from_settings(_cfg(image_fallbacks="", video_fallbacks="")) assert r.chain(Scene.CHARACTER_ACTION) == ("kling-v2-5-turbo",) + + +def test_image_alt_can_be_fallback(): + r = ModelRegistry.from_settings(_cfg(image_fallbacks="gemini-2.5-flash-image-alt")) + assert r.chain(Scene.CHARACTER_IMAGE) == ( + "gemini-2.5-flash-image", + "gemini-2.5-flash-image-alt", + ) + assert r.family_of("gemini-2.5-flash-image-alt") is Family.IMAGE_CHAT_DATA_URI From ef2363aebe042f8c1ff55dac87e430faa2a970ca Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:12:52 +0800 Subject: [PATCH 08/15] =?UTF-8?q?feat(gateway):=20=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E5=BB=BA=E5=8D=95=E4=B8=8E=E8=B7=9F=E5=8D=95=E5=88=86=E7=A6=BB?= =?UTF-8?q?=E5=B9=B6=E6=8C=89=E7=AD=96=E7=95=A5=20Fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/video.py | 402 ++++++++++++++++++ .../src/windup_framework/providers/sufy.py | 156 ++++++- backend/tests/test_gateway_video.py | 76 ++++ backend/tests/test_sufy_video_download.py | 15 +- 4 files changed, 629 insertions(+), 20 deletions(-) create mode 100644 backend/packages/framework/src/windup_framework/gateway/video.py create mode 100644 backend/tests/test_gateway_video.py diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py new file mode 100644 index 00000000..6e2f0249 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +import time +import uuid +from datetime import datetime, timezone +from urllib.parse import urlparse + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.image import _CIRCUIT +from windup_framework.gateway.policy import decide +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.trace import ( + AttemptTrace, + emit, + estimate_cost, + hash_bytes, + hash_image_input, +) +from windup_framework.gateway.types import NextStep, Scene + +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class VideoGateway: + def __init__(self, registry, adapter, circuit, settings) -> None: + self._registry = registry + self._adapter = adapter + self._circuit = circuit + self._settings = settings + + def i2v( + self, + first_frame: bytes, + prompt: str, + seconds: int = 5, + size: str = "1280x720", + ) -> bytes: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = hash_image_input(prompt, [first_frame]) + host = urlparse(self._settings.base_url).hostname + last_http_status: int | None = None + last_error: ModelErrorType | None = None + fallback_used = False + fallback_reason: str | None = None + + chain = list(self._registry.chain(Scene.CHARACTER_ACTION)) + if ctx.start_from_model and ctx.start_from_model in chain: + start_i = chain.index(ctx.start_from_model) + models = chain[start_i:] + else: + start_i = 0 + models = chain + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + err = last_error.value if last_error is not None else None + raise RuntimeError( + f"video gateway failed request_id={request_id} " + f"http_status={http_status} error_type={err}" + ) + + if self._circuit.is_open("aggregator"): + model = models[0] if models else "" + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="aggregator", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=False, + ) + fail(None) + + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="model", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + ) + continue + + if i == 0: + route_reason = ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" + ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + bound_job_id: str | None = None + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + if bound_job_id is None: + result = self._adapter.submit_video( + first_frame, prompt, seconds, size, model + ) + if result.ok and result.job_id: + bound_job_id = result.job_id + result = self._adapter.follow_job(bound_job_id) + elif result.ok: + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + self._emit_result( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="fallback_success" if fallback_used else "success", + ) + return result.body + else: + result = self._adapter.follow_job(bound_job_id) + + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + if result.ok: + self._emit_result( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="fallback_success" if fallback_used else "success", + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + last_error = error_type + has_job_id = bool(result.job_id or bound_job_id) + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=has_job_id, + ) + circuit_scope = None + if step is NextStep.OPEN_AGGREGATOR: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit_result( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type, + ) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + bound_job_id = None + break + fail(last_http_status) + + fail(last_http_status) + + def _emit_result( + self, + *, + request_id: str, + ctx, + model: str, + host: str | None, + attempt_index: int, + retry_count: int, + route_reason: str, + result, + input_hash: str, + total_latency_ms: int, + fallback_used: bool, + started_at: str, + ended_at: str, + attempt_latency_ms: int, + resend_spent: int, + seconds: int, + outcome: str, + circuit_scope: str | None = None, + error_type: ModelErrorType | None = None, + ) -> None: + billed = result.ok or result.maybe_billed + cost = estimate_cost( + Scene.CHARACTER_ACTION, + billed=billed, + seconds=seconds, + image_unit_cost=self._settings.image_unit_cost, + video_unit_cost_per_second=self._settings.video_unit_cost_per_second, + ) + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=circuit_scope, + outcome=outcome, + input_hash=input_hash, + output_hash=hash_bytes(result.body) if result.ok else None, + total_latency_ms=total_latency_ms, + fallback_used=fallback_used, + http_status=result.http_status, + error_type=error_type.value if error_type is not None else None, + maybe_billed=True if result.ok else result.maybe_billed, + cost=cost, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + output_bytes=len(result.body) if result.ok else (result.output_bytes or None), + expected_bytes=result.expected_bytes, + provider_usage=result.provider_usage, + edge_fingerprint=result.edge_fingerprint or None, + job_id=result.job_id, + job_status=result.job_status, + retry_after_ms=retry_after_ms, + ) + + def _emit( + self, + *, + request_id: str, + ctx, + model: str, + host: str | None, + attempt_index: int, + retry_count: int, + route_reason: str, + circuit_scope: str | None, + outcome: str, + input_hash: str, + total_latency_ms: int, + fallback_used: bool, + output_hash: str | None = None, + http_status: int | None = None, + error_type: str | None = None, + maybe_billed: bool | None = None, + cost: float | None = None, + started_at: str | None = None, + ended_at: str | None = None, + attempt_latency_ms: int | None = None, + resend_spent: int | None = 0, + output_bytes: int | None = None, + expected_bytes: int | None = None, + provider_usage: object | None = None, + edge_fingerprint: str | None = None, + job_id: str | None = None, + job_status: str | None = None, + retry_after_ms: int | None = None, + ) -> None: + family = None + if model: + family = self._registry.family_of(model).value + emit( + AttemptTrace( + request_id=request_id, + attempt_id=str(uuid.uuid4()), + task_id=ctx.task_id, + user_id=ctx.user_id, + scene=Scene.CHARACTER_ACTION, + model=model, + family=family, + base_url_host=host, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=circuit_scope, + error_type=error_type, + http_status=http_status, + edge_fingerprint=edge_fingerprint, + job_id=job_id, + fallback_used=fallback_used, + outcome=outcome, + job_status=job_status, + started_at=started_at or _utc_now(), + ended_at=ended_at or _utc_now(), + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_latency_ms, + submit_ms=None, + poll_ms=None, + download_ms=None, + poll_count=None, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + output_bytes=output_bytes, + expected_bytes=expected_bytes, + input_hash=input_hash, + output_hash=output_hash, + maybe_billed=maybe_billed, + cost=cost, + price_version=self._settings.price_version, + provider_usage=provider_usage, + ) + ) + + +def build_video_gateway(config=None, *, adapter=None, circuit=None) -> VideoGateway: + cfg: AIProviderSettings = config or default_settings + if adapter is None: + from windup_framework.providers.sufy import SufyVideoProvider + + adapter = SufyVideoProvider(config=cfg) + return VideoGateway( + ModelRegistry.from_settings(cfg), + adapter, + circuit if circuit is not None else _CIRCUIT, + cfg, + ) diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index ea27deb5..98e3556c 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -70,6 +70,31 @@ def _first_frame_datauri(frame: bytes, size: str) -> str: return "data:image/jpeg;base64," + base64.b64encode(_fit_first_frame(frame, size)).decode() +def _video_http_error(resp: httpx.Response, *, job_id: str | None = None) -> AdapterResult: + error_type = classify_http(resp.status_code) + retry_after_header = resp.headers.get("Retry-After") + retry_after_s = ( + _retry_after_seconds(retry_after_header) if retry_after_header else None + ) + return AdapterResult( + ok=False, + error_type=error_type, + http_status=resp.status_code, + maybe_billed=job_id is not None or error_type is ModelErrorType.MAYBE_BILLED, + edge_fingerprint=_edge_fingerprint(resp), + retry_after_s=retry_after_s, + job_id=job_id, + ) + + +def _poll_get(client: httpx.Client, job_id: str) -> httpx.Response: + """轮询 GET;522/525(及同档未达上游码)该次再试 1 次,不新开单。""" + resp = client.get(f"/videos/{job_id}") + if resp.status_code in (521, 522, 523, 525): + resp = client.get(f"/videos/{job_id}") + return resp + + class SufyVideoProvider(VideoProvider): """kling i2v(默认 v2-5-turbo)。首帧 + 动作 prompt → mp4 bytes。""" @@ -99,39 +124,142 @@ def _client(self) -> httpx.Client: timeout=self._cfg.timeout, ) - def i2v( - self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" - ) -> bytes: + def submit_video( + self, + first_frame: bytes, + prompt: str, + seconds: int, + size: str, + model: str, + ) -> AdapterResult: + """一次 POST 建单。成功: ok=True, job_id, body=b"", maybe_billed=True。""" body: dict = { - "model": self._model, + "model": model, "prompt": prompt, "size": size, "seconds": str(seconds), "mode": self._mode, } - if self._model in _IMAGE_LIST_MODELS: + if model in _IMAGE_LIST_MODELS: b64 = _first_frame_datauri(first_frame, size).split(",", 1)[1] body["image_list"] = [{"image": b64}] else: body["input_reference"] = _first_frame_datauri(first_frame, size) with self._client() as client: - job = client.post("/videos", json=body).raise_for_status().json() - jid = job.get("id") + resp = client.post("/videos", json=body) + + if 200 <= resp.status_code < 300: + try: + payload = resp.json() + except ValueError: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应不是 JSON", + ) + jid = payload.get("id") + if not jid: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应没有 job id", + ) + return AdapterResult( + ok=True, + job_id=str(jid), + body=b"", + maybe_billed=True, + http_status=resp.status_code, + ) + return _video_http_error(resp) + + def follow_job(self, job_id: str) -> AdapterResult: + """轮询已建单据 + 下载。poll GET 522/525 该次再试 1 次,不新开单。""" + with self._client() as client: url = None + last_status: str | None = None for _ in range(max(1, int(self._max_min * 60 // self._poll))): time.sleep(self._poll) - st = client.get(f"/videos/{jid}").raise_for_status().json() - status = st.get("status") - if status == "completed": + resp = _poll_get(client, job_id) + if not (200 <= resp.status_code < 300): + return _video_http_error(resp, job_id=job_id) + try: + st = resp.json() + except ValueError: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + job_id=job_id, + maybe_billed=True, + edge_fingerprint="轮询响应不是 JSON", + ) + last_status = st.get("status") + if last_status == "completed": vids = (st.get("task_result") or {}).get("videos") or [] url = vids[0].get("url") if vids else None break - if status in ("failed", "cancelled"): - raise RuntimeError(f"i2v 失败: {status} — {st.get('error')}") + if last_status in ("failed", "cancelled"): + return AdapterResult( + ok=False, + error_type=ModelErrorType.UPSTREAM_FAILED, + job_id=job_id, + maybe_billed=True, + job_status=last_status, + edge_fingerprint=str(st.get("error") or ""), + ) if not url: - raise RuntimeError("i2v 未取得视频 URL(超时或失败)") - return _download(client, url) + return AdapterResult( + ok=False, + error_type=ModelErrorType.TIMEOUT, + job_id=job_id, + maybe_billed=True, + job_status=last_status or "timeout", + ) + try: + body = _download(client, url) + except RuntimeError as exc: + return AdapterResult( + ok=False, + error_type=ModelErrorType.MAYBE_BILLED, + job_id=job_id, + maybe_billed=True, + job_status="completed", + edge_fingerprint=str(exc), + ) + return AdapterResult( + ok=True, + body=body, + job_id=job_id, + maybe_billed=True, + job_status="completed", + ) + + def i2v( + self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" + ) -> bytes: + submitted = self.submit_video(first_frame, prompt, seconds, size, self._model) + if not submitted.ok or not submitted.job_id: + raise RuntimeError( + f"i2v 建单失败(HTTP {submitted.http_status} {submitted.error_type}): " + f"{submitted.edge_fingerprint}" + ) + followed = self.follow_job(submitted.job_id) + if followed.ok: + return followed.body + if followed.error_type is ModelErrorType.TIMEOUT: + raise RuntimeError("i2v 未取得视频 URL(超时或失败)") + if followed.error_type is ModelErrorType.UPSTREAM_FAILED: + raise RuntimeError( + f"i2v 失败: {followed.job_status} — {followed.edge_fingerprint}" + ) + raise RuntimeError( + f"i2v 失败(HTTP {followed.http_status} {followed.error_type}): " + f"{followed.edge_fingerprint}" + ) class IncompleteDownloadError(RuntimeError): diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py new file mode 100644 index 00000000..2dea020a --- /dev/null +++ b/backend/tests/test_gateway_video.py @@ -0,0 +1,76 @@ +import pytest +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import AdapterResult +from windup_framework.gateway.video import VideoGateway + +UNREACHED = AdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +FAILED_JOB = AdapterResult( + ok=False, error_type=ModelErrorType.UPSTREAM_FAILED, job_id="j1", maybe_billed=True, +) +TIMEOUT = AdapterResult(ok=False, error_type=ModelErrorType.TIMEOUT, job_id="j1", maybe_billed=True) +MP4 = AdapterResult(ok=True, body=b"\x00\x00\x00\x18ftypmp42", maybe_billed=True) + +class FakeVideoAdapter: + def __init__(self, submits: dict[str, list[AdapterResult]], follows: dict[str, AdapterResult]): + self.submits = {k: list(v) for k, v in submits.items()} + self.follows = dict(follows) + self.submit_models: list[str] = [] + self.followed: list[str] = [] + + def submit_video(self, first_frame, prompt, seconds, size, model): + self.submit_models.append(model) + return self.submits[model].pop(0) + + def follow_job(self, job_id): + self.followed.append(job_id) + return self.follows[job_id] + +def _video_gw(adapter) -> VideoGateway: + cfg = AIProviderSettings(video_model="kling-v2-5-turbo", video_fallbacks="kling-v2-6") + return VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=adapter, + circuit=CircuitBreaker(cooldown_s=60), + settings=cfg, + ) + +def test_submit_522_retries_once_does_not_open_second_job_on_fallback_model(): + ad = FakeVideoAdapter( + submits={"kling-v2-5-turbo": [UNREACHED, UNREACHED], "kling-v2-6": [ + AdapterResult(ok=True, job_id="j-alt", maybe_billed=True) + ]}, + follows={}, + ) + with pytest.raises(RuntimeError, match="522"): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo", "kling-v2-5-turbo"] + assert ad.followed == [] + +def test_follow_failed_opens_new_job_on_fallback(): + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": FAILED_JOB, "j2": MP4}, + ) + body = _video_gw(ad).i2v(b"frame", "walk") + assert body.startswith(b"\x00\x00\x00\x18ftyp") + assert ad.submit_models == ["kling-v2-5-turbo", "kling-v2-6"] + assert ad.followed == ["j1", "j2"] + +def test_timeout_does_not_submit_fallback(): + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": TIMEOUT}, + ) + with pytest.raises(RuntimeError, match="timeout|超时"): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo"] + assert ad.followed == ["j1"] diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 957e9fec..a4840c15 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -682,12 +682,15 @@ def _video_provider(handler, **kw): poll_interval=30.0, **kw, ) - client = _httpx.Client( - base_url="https://gw.example.com/v1", - headers={"Authorization": "Bearer k"}, - transport=_httpx.MockTransport(handler), - ) - p._client = lambda: client + # submit_video 与 follow_job 各自 with _client()，必须每次返回新 client， + # 否则建单结束就会把同一实例 close 掉，跟单 GET 打到已关闭的连接。 + def make_client(): + return _httpx.Client( + base_url="https://gw.example.com/v1", + headers={"Authorization": "Bearer k"}, + transport=_httpx.MockTransport(handler), + ) + p._client = make_client return p From 7cef41f530c6e41c2ea46a839a21edf91b70d884 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:18:17 +0800 Subject: [PATCH 09/15] =?UTF-8?q?fix(gateway):=20=E5=B7=B2=E6=9C=89=20job?= =?UTF-8?q?=5Fid=20=E6=97=B6=E7=A6=81=E6=AD=A2=20429=20=E5=86=8D=E5=BC=80?= =?UTF-8?q?=E7=AC=AC=E4=BA=8C=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../src/windup_framework/gateway/video.py | 5 ++++ backend/tests/test_gateway_video.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index 6e2f0249..48ad9891 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -233,6 +233,11 @@ def fail(http_status: int | None) -> None: resend_spent = 1 continue if step is NextStep.FALLBACK: + if ( + bound_job_id is not None + and error_type is not ModelErrorType.UPSTREAM_FAILED + ): + fail(last_http_status) fallback_used = True fallback_reason = ( "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py index 2dea020a..395cd965 100644 --- a/backend/tests/test_gateway_video.py +++ b/backend/tests/test_gateway_video.py @@ -74,3 +74,28 @@ def test_timeout_does_not_submit_fallback(): _video_gw(ad).i2v(b"frame", "walk") assert ad.submit_models == ["kling-v2-5-turbo"] assert ad.followed == ["j1"] + + +@pytest.mark.parametrize( + "error_type", + [ModelErrorType.RATE_LIMIT, ModelErrorType.INVALID_RESPONSE], +) +def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_type): + follow_result = AdapterResult( + ok=False, + error_type=error_type, + job_id="j1", + retry_after_s=0, + maybe_billed=True, + ) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": follow_result}, + ) + with pytest.raises(RuntimeError, match=error_type.value): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo"] + assert ad.followed == ["j1", "j1", "j1"] From 4a9fe82352692d93f75158dfce9d015122261f7f Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:27:23 +0800 Subject: [PATCH 10/15] =?UTF-8?q?feat(gateway):=20=E7=94=9F=E6=88=90?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=BB=8F=20Gateway=20=E8=A3=85=E9=85=8D?= =?UTF-8?q?=E5=B9=B6=E5=9B=9E=E5=86=99=20request=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- .../server/orchestrator/executor.py | 103 +++++++------ .../windup_app/server/orchestrator/model.py | 6 +- .../app/src/windup_app/web/api/generation.py | 4 +- .../src/windup_framework/gateway/__init__.py | 10 +- .../src/windup_framework/gateway/registry.py | 5 +- .../windup_framework/providers/__init__.py | 5 + backend/tests/test_custom_action.py | 44 +++--- backend/tests/test_gateway_executor.py | 145 ++++++++++++++++++ 8 files changed, 238 insertions(+), 84 deletions(-) create mode 100644 backend/tests/test_gateway_executor.py diff --git a/backend/packages/app/src/windup_app/server/orchestrator/executor.py b/backend/packages/app/src/windup_app/server/orchestrator/executor.py index 7459e6f7..b16bcd2c 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/executor.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/executor.py @@ -22,6 +22,9 @@ from sqlalchemy.orm import Session from windup_common.models import ActionSpec, ActionType as EngineActionType, CharacterCard +from windup_framework.gateway import bind_call_context +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import Scene from windup_app.server.orchestrator import task_repo from windup_app.server.orchestrator._fetch import fetch_own_media @@ -140,27 +143,17 @@ def step(self, stage: str, i: int, total: int, note: str = "") -> None: logger.info("[gen] %s %s/%s %s", stage, i, total, note) -# 白名单而不是放开任意模型名:每个模型的入参形状不同(image_list / input_reference / -# Fal 队列 + `Authorization: Key`)。列进来却没适配它的协议,等于"看起来能选、点了必然 -# 产生一个用不了的付费任务"。只列 SufyVideoProvider 真能建单的。Refs #239。 -ALLOWED_VIDEO_MODELS: dict[str, str] = { - "kling-v2-5-turbo": "默认。稳,本地首帧即可", - "kling-v2-6": "有 motion-control", -} - - def _resolve_video_model(name: str | None) -> str | None: """校验并返回视频模型名;``None`` 表示用部署默认值。 + 只允许是 CHARACTER_ACTION 链上的一员,含义是「这次从它开始试」。 非法取值在入口炸,不等到付费调用才失败。 """ if name is None: return None - if name not in ALLOWED_VIDEO_MODELS: - raise ValueError( - f"视频模型 {name!r} 不在本期开放列表内。可选:" - + "；".join(f"{k}({v})" for k, v in ALLOWED_VIDEO_MODELS.items()) - ) + chain = ModelRegistry.from_settings().chain(Scene.CHARACTER_ACTION) + if name not in chain: + raise ValueError(f"视频模型 {name!r} 不在本期开放列表内。可选:" + "；".join(chain)) return name @@ -188,11 +181,9 @@ def __init__( fetch_constraints: Callable[[Session, int | None], ProjectConstraints] | None = None, session_factory: Callable[[], Session] | None = None, ) -> None: - self._generator = generator # None → 懒加载真实装配 - # 按视频模型名分桶的 generator 缓存(模型是 provider 的构造参数,不能事后换) - self._by_model: dict[str | None, CharacterGeneratorPort] = {} - # 抠图 / 图生图 provider 与视频模型无关,所有模型桶共用一份:每个抠图实例都会 - # 各自惰性加载一份 ONNX 会话,按桶各建等于把同一个模型在进程里装多次。 + self._generator = generator # None → 懒加载真实装配(一套共享 Gateway) + # 抠图 / 图生图与视频 Gateway 无关型号分桶:选哪个 kling 是 Gateway 读 + # start_from_model 的事。每个抠图实例都会惰性加载一份 ONNX 会话,只装一次。 self._matte: MatteProvider | None = None self._image: ImageProvider | None = None # 本执行器是进程级单例,而每个请求起一个线程跑 run_action_task,上面几个缓存 @@ -216,14 +207,21 @@ def run_action_task( 先从 ``project`` 取全局约束(朝向/画风/尺寸/方向)再调 ai_engine。``session`` 缺省时自开一个(后台场景);测试可传入自己的 session。 """ + request_id = f"act-{task_id}" own = session is None session = session or self._make_session() + reset = None try: task_repo.update_status(session, task_id, TaskStatus.RUNNING) if own: session.commit() cons = (self._fetch_constraints or _load_constraints)(session, project_id) + reset = bind_call_context( + request_id=request_id, + task_id=str(task_id), + start_from_model=_resolve_video_model(input.video_model), + ) result = self._produce_action(input, cons) task_repo.update_result(session, task_id, _ACTION_RESULT, result) if own: @@ -231,11 +229,14 @@ def run_action_task( except Exception as exc: # noqa: BLE001 —— 兜底任何生成/上传/网络异常 logger.exception("动作任务 %s 失败", task_id) task_repo.update_status( - session, task_id, TaskStatus.FAILED, error_message=str(exc), + session, task_id, TaskStatus.FAILED, + error_message=f"{exc}; request_id={request_id}", ) if own: session.commit() finally: + if reset is not None: + reset() if own: session.close() @@ -281,7 +282,7 @@ def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) **extra, ) progress: ProgressPort = _LogProgress() - generated = self._get_generator(_resolve_video_model(input.video_model)).generate( + generated = self._get_generator().generate( card, action, master, progress, canvas=(cons.sprite_w, cons.sprite_h) ) @@ -294,46 +295,37 @@ def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) ] return {"type": "character_action", "action_type": input.action_type.value, "frames": frames} - def _get_generator(self, video_model: str | None = None) -> CharacterGeneratorPort: - """懒装配 CharacterGenerator,按模型名分桶。 + def _get_generator(self) -> CharacterGeneratorPort: + """懒装配一套共享 CharacterGenerator(ImageGateway + VideoGateway + matte)。 - 视频 provider 的模型是构造参数,不分桶的话第一个请求指定的模型会被后续所有请求 - 沿用,而调用方以为自己指定了。 + 选哪个 kling 不在装配时定,由 bind_call_context 的 start_from_model 交给 Gateway。 """ if self._generator is not None: return self._generator - # 命中缓存的快路径不进锁,否则每个请求都要在这里排一次队。只有装配新桶才上锁, - # 锁内重查一次:两个线程同时错过同一个桶时,后进来的那个要看见前一个的成果。 - cached = self._by_model.get(video_model) - if cached is not None: - return cached + # 命中缓存的快路径不进锁,否则每个请求都要在这里排一次队。只有首次装配才上锁, + # 锁内重查一次:两个线程同时错过时,后进来的那个要看见前一个的成果。 with self._assembly_lock: - cached = self._by_model.get(video_model) - if cached is None: - cached = self._assemble(video_model) - self._by_model[video_model] = cached - return cached - - def _assemble(self, video_model: str | None) -> CharacterGeneratorPort: - """装一个模型桶。**调用方须持有 ``self._assembly_lock``**(会写共用 provider)。""" + if self._generator is None: + self._generator = self._assemble() + return self._generator + + def _assemble(self) -> CharacterGeneratorPort: + """装一套共享 Gateway。**调用方须持有 ``self._assembly_lock``**。""" from windup_ai_engine.impl import CharacterGenerator from windup_ai_engine.strategy.concrete import ( PerFrameStrategy, VideoFrameStrategy, ) from windup_common.models import GenRoute - from windup_framework.providers import ( - OnnxU2NetMatteProvider, - SufyImageProvider, - SufyVideoProvider, - ) + from windup_framework.gateway import build_image_gateway, build_video_gateway + from windup_framework.gateway.image import _CIRCUIT + from windup_framework.providers import OnnxU2NetMatteProvider if self._matte is None: self._matte = OnnxU2NetMatteProvider() if self._image is None: - self._image = SufyImageProvider() - # 只有它随模型变 —— 模型是构造参数,换模型必须换实例。 - video = SufyVideoProvider(model=video_model) + self._image = build_image_gateway(circuit=_CIRCUIT) + video = build_video_gateway(circuit=_CIRCUIT) # 装配表必须与 GenRoute 对齐。下面那条断言让漏装在装配时暴露,而不是等到某个 # 动作第一次被请求时才炸——注入 generator 的测试走不到这条装配路径,漏了会测试 # 全绿而真实调用全崩。 @@ -385,7 +377,7 @@ class ImageTaskExecutor: def __init__( self, *, - image=None, # None → 懒加载 SufyImageProvider + image=None, # None → 懒加载 ImageGateway upload: Callable[[bytes], str] | None = None, # None → 真实对象存储上传 fetch_ref: Callable[[str], bytes] | None = None, # None → 下载 reference_image_url session_factory: Callable[[], Session] | None = None, @@ -403,13 +395,19 @@ def run_image_task( *, session: Session | None = None, ) -> None: + request_id = f"img-{task_id}" own = session is None session = session or self._make_session() + reset = None try: task_repo.update_status(session, task_id, TaskStatus.RUNNING) if own: session.commit() cons = _load_constraints(session, project_id) # 角色图也受项目约束 + reset = bind_call_context( + request_id=request_id, + task_id=str(task_id), + ) urls = self._produce_image(input, cons) task_repo.update_result(session, task_id, _IMAGE_RESULT, { "type": "character_image", @@ -419,10 +417,15 @@ def run_image_task( session.commit() except Exception as exc: # noqa: BLE001 —— 兜底 logger.exception("图片任务 %s 失败", task_id) - task_repo.update_status(session, task_id, TaskStatus.FAILED, error_message=str(exc)) + task_repo.update_status( + session, task_id, TaskStatus.FAILED, + error_message=f"{exc}; request_id={request_id}", + ) if own: session.commit() finally: + if reset is not None: + reset() if own: session.close() @@ -484,9 +487,9 @@ def _produce_image(self, input: CharacterImageInput, cons: ProjectConstraints) - def _get_image(self): if self._image is None: - from windup_framework.providers import SufyImageProvider + from windup_framework.gateway import build_image_gateway - self._image = SufyImageProvider() + self._image = build_image_gateway() return self._image def _download(self, url: str) -> bytes: diff --git a/backend/packages/app/src/windup_app/server/orchestrator/model.py b/backend/packages/app/src/windup_app/server/orchestrator/model.py index 3c4aab8e..0b59d98b 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/model.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/model.py @@ -74,9 +74,9 @@ class CharacterActionInput: # 这个动作是否循环播放。``None`` 原样往下传,由编排层兜成一次性:本层替调用方填默认值 # 的话,"没给"和"明确给了 False"从这里起就再也分不开了。 loop: bool | None = None - # 视频模型。``None`` = 用部署配置的默认值(kling-v2-5-turbo)。 - # 取值域见 executor.ALLOWED_VIDEO_MODELS —— 只开放两个,因为每个模型的入参形状不同 - # (image_list / input_reference / Fal 队列),全开等于把三套协议适配塞进一个改动。 + # 视频模型。``None`` = 用部署配置的默认值。取值域为 + # ``ModelRegistry.chain(CHARACTER_ACTION)``(部署默认 + fallbacks);不在链上 → 入口 + # 报错,不到付费调用才失败。选中的型号表示这次从它开始试,由 Gateway 读 start_from_model。 video_model: str | None = None diff --git a/backend/packages/app/src/windup_app/web/api/generation.py b/backend/packages/app/src/windup_app/web/api/generation.py index dc5c2907..c5a6c92b 100644 --- a/backend/packages/app/src/windup_app/web/api/generation.py +++ b/backend/packages/app/src/windup_app/web/api/generation.py @@ -179,8 +179,8 @@ class CharacterActionGenerateRequest(BaseModel): # 不对称:一次性动作被当成循环会让末帧接回首帧抽搐、产物不可用,反之只是不无缝闭环、 # 仍可用。而且猜错是静默的,帧数/时长/成色全部正常、没有任何一道会红。 loop: bool | None = None - # 视频模型。None = 用部署默认(kling-v2-5-turbo)。取值域见 - # orchestrator.executor.ALLOWED_VIDEO_MODELS;非法值在入口就报错,不到付费调用才失败。 + # 视频模型。None = 用部署默认。取值域见 ModelRegistry.chain(CHARACTER_ACTION); + # 非法值在入口就报错,不到付费调用才失败。选中的型号表示这次从它开始试。 video_model: str | None = None @model_validator(mode="after") diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py index 084564d7..17dbf637 100644 --- a/backend/packages/framework/src/windup_framework/gateway/__init__.py +++ b/backend/packages/framework/src/windup_framework/gateway/__init__.py @@ -1,3 +1,11 @@ +from windup_framework.gateway.context import bind_call_context from windup_framework.gateway.image import ImageGateway, build_image_gateway +from windup_framework.gateway.video import VideoGateway, build_video_gateway -__all__ = ["ImageGateway", "build_image_gateway"] +__all__ = [ + "ImageGateway", + "VideoGateway", + "bind_call_context", + "build_image_gateway", + "build_video_gateway", +] diff --git a/backend/packages/framework/src/windup_framework/gateway/registry.py b/backend/packages/framework/src/windup_framework/gateway/registry.py index f6ac2564..77d3ee3d 100644 --- a/backend/packages/framework/src/windup_framework/gateway/registry.py +++ b/backend/packages/framework/src/windup_framework/gateway/registry.py @@ -1,6 +1,6 @@ from __future__ import annotations -from windup_framework.config.provider import AIProviderSettings +from windup_framework.config.provider import AIProviderSettings, settings as default_settings from windup_framework.gateway.types import Family, Scene FAMILIES: dict[str, Family] = { @@ -25,7 +25,8 @@ def __init__(self, chains: dict[Scene, tuple[str, ...]]) -> None: self._chains = chains @classmethod - def from_settings(cls, cfg: AIProviderSettings) -> ModelRegistry: + def from_settings(cls, cfg: AIProviderSettings | None = None) -> ModelRegistry: + cfg = default_settings if cfg is None else cfg chains = { Scene.CHARACTER_IMAGE: (cfg.image_model, *_parse_fallbacks(cfg.image_fallbacks)), Scene.CHARACTER_ACTION: (cfg.video_model, *_parse_fallbacks(cfg.video_fallbacks)), diff --git a/backend/packages/framework/src/windup_framework/providers/__init__.py b/backend/packages/framework/src/windup_framework/providers/__init__.py index fd1f448e..f402c586 100644 --- a/backend/packages/framework/src/windup_framework/providers/__init__.py +++ b/backend/packages/framework/src/windup_framework/providers/__init__.py @@ -1,6 +1,7 @@ """按模型能力划分的 AI Provider:官方客户端工厂 + 能力接口 + SUFY 实现。""" from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway import bind_call_context, build_image_gateway, build_video_gateway from windup_framework.providers.chat import create_chat_model from windup_framework.providers.image import create_image_client from windup_framework.providers.interfaces import ( @@ -29,4 +30,8 @@ # FAL 队列面的 i2v(现役接口形态);首帧要公网 URL,故与 uploader 成对出现 "SufyImageProvider", "OnnxU2NetMatteProvider", + # Gateway 工厂(executor 从 windup_framework.gateway 取;此处再导出方便装配) + "bind_call_context", + "build_image_gateway", + "build_video_gateway", ] diff --git a/backend/tests/test_custom_action.py b/backend/tests/test_custom_action.py index b682d291..131414aa 100644 --- a/backend/tests/test_custom_action.py +++ b/backend/tests/test_custom_action.py @@ -228,19 +228,12 @@ def test_illegal_facing_raises_instead_of_falling_back(): def test_only_the_opened_models_are_accepted(): - from windup_app.server.orchestrator.executor import ( - ALLOWED_VIDEO_MODELS, - _resolve_video_model, - ) + from windup_framework.config.provider import AIProviderSettings + from windup_framework.gateway.registry import ModelRegistry + from windup_framework.gateway.types import Scene - # veo3.1 不在表里:它走 Fal 队列协议(Authorization: Key + 公网图 URL),而 - # SufyVideoProvider 走 OpenAI 风格 /videos + Bearer + base64。列进去 = 看起来能选、 - # 点了必然产生一个用不了的付费任务。 - assert set(ALLOWED_VIDEO_MODELS) == {"kling-v2-5-turbo", "kling-v2-6"} - assert "veo3.1" not in ALLOWED_VIDEO_MODELS - for name in ALLOWED_VIDEO_MODELS: - assert _resolve_video_model(name) == name - assert _resolve_video_model(None) is None, "None = 用部署默认值" + r = ModelRegistry.from_settings(AIProviderSettings(video_fallbacks="kling-v2-6")) + assert set(r.chain(Scene.CHARACTER_ACTION)) == {"kling-v2-5-turbo", "kling-v2-6"} def test_unknown_model_fails_at_entry_not_at_the_paid_call(): @@ -252,23 +245,21 @@ def test_unknown_model_fails_at_entry_not_at_the_paid_call(): assert "kling-v2-5-turbo" in str(e.value), "报错要带上可选值,否则调用方无从改" -def test_generator_is_bucketed_by_video_model(): - """按模型分桶,否则第一个请求指定 veo3.1 之后所有请求都沿用它。""" +def test_start_from_model_reuses_one_generator(): from windup_app.server.orchestrator.executor import ActionTaskExecutor ex = ActionTaskExecutor() - a = ex._get_generator("kling-v2-6") - b = ex._get_generator("veo3.1") - assert a is not b, "两个模型拿到了同一个 generator" - assert ex._get_generator("kling-v2-6") is a, "同一模型该复用" + assert ex._get_generator() is ex._get_generator() def test_concurrent_first_requests_build_one_shared_provider_set(monkeypatch): - """并发首请求只装一份共用 provider。 + """并发首请求只装一份共用 Gateway / matte。 执行器是进程级单例、每个请求起一个线程,check-and-insert 不加锁时每个线程都会各装 一套;而每个抠图实例会各自惰性加载一份 ONNX 会话,重复的代价落在内存与加载耗时上。 + 选哪个 kling 是 Gateway 的事,不同 video_model 仍共用同一个 generator。 """ + import windup_framework.gateway as gateway from windup_framework import providers from windup_app.server.orchestrator.executor import ActionTaskExecutor @@ -285,8 +276,8 @@ def _factory(*_args, **_kwargs): return _factory monkeypatch.setattr(providers, "OnnxU2NetMatteProvider", _counting("matte")) - monkeypatch.setattr(providers, "SufyImageProvider", _counting("image")) - monkeypatch.setattr(providers, "SufyVideoProvider", _counting("video")) + monkeypatch.setattr(gateway, "build_image_gateway", _counting("image")) + monkeypatch.setattr(gateway, "build_video_gateway", _counting("video")) ex = ActionTaskExecutor() models = ["kling-v2-5-turbo", "kling-v2-6"] * 3 @@ -295,7 +286,7 @@ def _factory(*_args, **_kwargs): def _ask(i: int) -> None: start.wait(timeout=5) - gen = ex._get_generator(models[i]) + gen = ex._get_generator() with tally: got[i] = gen @@ -307,10 +298,11 @@ def _ask(i: int) -> None: assert not any(t.is_alive() for t in threads), "有线程没跑完,装配路径可能卡在锁上" assert built.count("matte") == 1, f"抠图 provider 装了 {built.count('matte')} 次,该只装一次" - assert built.count("image") == 1, f"图生图 provider 装了 {built.count('image')} 次" - assert built.count("video") == 2, "视频 provider 随模型变,两个模型该各一份" - for i, model in enumerate(models): - assert got[i] is ex._by_model[model], "同一模型的并发请求该拿到同一个 generator" + assert built.count("image") == 1, f"图生图 Gateway 装了 {built.count('image')} 次" + assert built.count("video") == 1, f"视频 Gateway 装了 {built.count('video')} 次,该只装一次" + gens = {got[i] for i in range(len(models))} + assert len(gens) == 1, "不同 video_model 的并发请求该拿到同一个 generator" + assert next(iter(gens)) is ex._get_generator() # ── ⑥ 骨架不得夹带姿态前提(游泳/潜水/飞行都不着地不直立)───────────────────── diff --git a/backend/tests/test_gateway_executor.py b/backend/tests/test_gateway_executor.py new file mode 100644 index 00000000..dc7e1132 --- /dev/null +++ b/backend/tests/test_gateway_executor.py @@ -0,0 +1,145 @@ +"""Executor 经 Gateway 装配,失败文案带回 request_id。""" +from __future__ import annotations + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from windup_framework.db.base import Base +from windup_framework.gateway.context import current_call_context +from windup_app.server.orchestrator.executor import ( + ActionTaskExecutor, + ImageTaskExecutor, + _resolve_video_model, +) +from windup_app.server.orchestrator.model import ( + ActionType, + CharacterActionInput, + CharacterImageInput, + TaskStatus, +) +from windup_app.server.orchestrator.service import AiGenerationService +from windup_app.server.project.model import Project # noqa: F401 — 注册表 + + +@pytest.fixture +def session_factory(): + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + return sessionmaker(bind=engine) + + +def test_none_video_model_means_deploy_default(): + assert _resolve_video_model(None) is None + + +def test_unknown_model_error_lists_chain_members(): + with pytest.raises(ValueError) as e: + _resolve_video_model("sora-2") + msg = str(e.value) + assert "sora-2" in msg + chain_hint = "kling-v2-5-turbo" + assert chain_hint in msg, "报错要带上链上型号,否则调用方无从改" + + +def test_action_task_failure_includes_request_id(session_factory): + seen: dict[str, str | None] = {} + + class _BoomGen: + def generate(self, *args, **kwargs): + ctx = current_call_context() + seen["request_id"] = ctx.request_id + seen["task_id"] = ctx.task_id + seen["start_from_model"] = ctx.start_from_model + raise RuntimeError("gateway boom") + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=_BoomGen(), + fetch_master=lambda _input: b"png", + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=4, + ) + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert f"request_id=act-{task_id}" in (done.error_message or "") + assert seen["request_id"] == f"act-{task_id}" + assert seen["task_id"] == str(task_id) + assert seen["start_from_model"] is None + + +def test_action_task_binds_start_from_model(session_factory): + seen: dict[str, str | None] = {} + + class _BoomGen: + def generate(self, *args, **kwargs): + seen["start_from_model"] = current_call_context().start_from_model + raise RuntimeError("boom") + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=_BoomGen(), + fetch_master=lambda _input: b"png", + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=4, + video_model="kling-v2-5-turbo", + ) + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert seen["start_from_model"] == "kling-v2-5-turbo" + assert f"request_id=act-{task_id}" in (done.error_message or "") + + +def test_image_task_failure_includes_request_id(session_factory): + seen: dict[str, str | None] = {} + + class _BoomImage: + def gen_image(self, prompt, refs): + seen["request_id"] = current_call_context().request_id + seen["task_id"] = current_call_context().task_id + raise RuntimeError("image boom") + + service = AiGenerationService() + executor = ImageTaskExecutor( + image=_BoomImage(), + session_factory=session_factory, + ) + image_input = CharacterImageInput(prompt="knight") + with session_factory() as s: + task = service.generate_character_image(s, user_id=1, input=image_input) + s.commit() + task_id = task.id + + executor.run_image_task(task_id, image_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert f"request_id=img-{task_id}" in (done.error_message or "") + assert seen["request_id"] == f"img-{task_id}" + assert seen["task_id"] == str(task_id) From 19fce6a717de0f430418f07b312a8e560d605bd2 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:47:46 +0800 Subject: [PATCH 11/15] =?UTF-8?q?fix(gateway):=20=E7=86=94=E6=96=AD?= =?UTF-8?q?=E5=8A=A0=E9=94=81=E5=B9=B6=E7=A6=81=E6=AD=A2=E6=97=A0=20job=5F?= =?UTF-8?q?id=20=E6=97=B6=E6=8D=A2=E5=9E=8B=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 并发过期不再 KeyError；跳过开路型号会记 fallback_used；视频分阶段耗时写入 trace；提交未拿到 job_id 时除 429 外不再换型号。 Co-authored-by: Cursor --- .../src/windup_framework/gateway/circuit.py | 20 ++-- .../src/windup_framework/gateway/image.py | 4 + .../src/windup_framework/gateway/types.py | 3 + .../src/windup_framework/gateway/video.py | 59 ++++++----- .../src/windup_framework/providers/sufy.py | 97 +++++++++++++------ backend/tests/test_gateway_image.py | 25 +++++ backend/tests/test_gateway_policy.py | 34 +++++++ backend/tests/test_gateway_video.py | 89 ++++++++++++++++- backend/tests/test_sufy_video_download.py | 9 ++ 9 files changed, 273 insertions(+), 67 deletions(-) diff --git a/backend/packages/framework/src/windup_framework/gateway/circuit.py b/backend/packages/framework/src/windup_framework/gateway/circuit.py index 1739e0fe..29680cb1 100644 --- a/backend/packages/framework/src/windup_framework/gateway/circuit.py +++ b/backend/packages/framework/src/windup_framework/gateway/circuit.py @@ -1,5 +1,6 @@ from __future__ import annotations +import threading import time from collections.abc import Callable @@ -14,15 +15,18 @@ def __init__( self._cooldown_s = cooldown_s self._monotonic = monotonic or time.monotonic self._open_until: dict[str, float] = {} + self._lock = threading.Lock() def is_open(self, key: str) -> bool: - until = self._open_until.get(key) - if until is None: - return False - if self._monotonic() >= until: - del self._open_until[key] - return False - return True + with self._lock: + until = self._open_until.get(key) + if until is None: + return False + if self._monotonic() >= until: + self._open_until.pop(key, None) + return False + return True def open(self, key: str) -> None: - self._open_until[key] = self._monotonic() + self._cooldown_s + with self._lock: + self._open_until[key] = self._monotonic() + self._cooldown_s diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py index e17e2b3c..ad294e09 100644 --- a/backend/packages/framework/src/windup_framework/gateway/image.py +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -83,6 +83,8 @@ def fail(http_status: int | None) -> None: for i, model in enumerate(models): attempt_index = start_i + i if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" self._emit( request_id=request_id, ctx=ctx, @@ -107,6 +109,8 @@ def fail(http_status: int | None) -> None: ) elif fallback_reason == "429": route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" else: route_reason = "fallback_after_upstream_fail" diff --git a/backend/packages/framework/src/windup_framework/gateway/types.py b/backend/packages/framework/src/windup_framework/gateway/types.py index 7e984ea6..d056238c 100644 --- a/backend/packages/framework/src/windup_framework/gateway/types.py +++ b/backend/packages/framework/src/windup_framework/gateway/types.py @@ -38,3 +38,6 @@ class AdapterResult: provider_usage: object | None = None job_status: str | None = None retry_after_s: float | None = None + poll_ms: int | None = None + download_ms: int | None = None + poll_count: int | None = None diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index 48ad9891..e45832e6 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -2,6 +2,7 @@ import time import uuid +from dataclasses import replace from datetime import datetime, timezone from urllib.parse import urlparse @@ -91,6 +92,8 @@ def fail(http_status: int | None) -> None: for i, model in enumerate(models): attempt_index = start_i + i if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" self._emit( request_id=request_id, ctx=ctx, @@ -115,6 +118,8 @@ def fail(http_status: int | None) -> None: ) elif fallback_reason == "429": route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" else: route_reason = "fallback_after_upstream_fail" @@ -124,37 +129,23 @@ def fail(http_status: int | None) -> None: while True: attempt_t0 = time.monotonic() started_at = _utc_now() + submit_ms: int | None = None if bound_job_id is None: + submit_t0 = time.monotonic() result = self._adapter.submit_video( first_frame, prompt, seconds, size, model ) + submit_ms = int((time.monotonic() - submit_t0) * 1000) if result.ok and result.job_id: bound_job_id = result.job_id result = self._adapter.follow_job(bound_job_id) elif result.ok: - ended_at = _utc_now() - attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) - last_http_status = result.http_status - self._emit_result( - request_id=request_id, - ctx=ctx, - model=model, - host=host, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - result=result, - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - seconds=seconds, - outcome="fallback_success" if fallback_used else "success", + result = replace( + result, + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + body=b"", ) - return result.body else: result = self._adapter.follow_job(bound_job_id) @@ -180,6 +171,7 @@ def fail(http_status: int | None) -> None: resend_spent=resend_spent, seconds=seconds, outcome="fallback_success" if fallback_used else "success", + submit_ms=submit_ms, ) return result.body @@ -219,6 +211,7 @@ def fail(http_status: int | None) -> None: outcome="failed", circuit_scope=circuit_scope, error_type=error_type, + submit_ms=submit_ms, ) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: @@ -238,6 +231,11 @@ def fail(http_status: int | None) -> None: and error_type is not ModelErrorType.UPSTREAM_FAILED ): fail(last_http_status) + if ( + bound_job_id is None + and error_type is not ModelErrorType.RATE_LIMIT + ): + fail(last_http_status) fallback_used = True fallback_reason = ( "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" @@ -270,6 +268,7 @@ def _emit_result( outcome: str, circuit_scope: str | None = None, error_type: ModelErrorType | None = None, + submit_ms: int | None = None, ) -> None: billed = result.ok or result.maybe_billed cost = estimate_cost( @@ -313,6 +312,10 @@ def _emit_result( job_id=result.job_id, job_status=result.job_status, retry_after_ms=retry_after_ms, + submit_ms=submit_ms, + poll_ms=result.poll_ms, + download_ms=result.download_ms, + poll_count=result.poll_count, ) def _emit( @@ -346,6 +349,10 @@ def _emit( job_id: str | None = None, job_status: str | None = None, retry_after_ms: int | None = None, + submit_ms: int | None = None, + poll_ms: int | None = None, + download_ms: int | None = None, + poll_count: int | None = None, ) -> None: family = None if model: @@ -375,10 +382,10 @@ def _emit( ended_at=ended_at or _utc_now(), attempt_latency_ms=attempt_latency_ms, total_latency_ms=total_latency_ms, - submit_ms=None, - poll_ms=None, - download_ms=None, - poll_count=None, + submit_ms=submit_ms, + poll_ms=poll_ms, + download_ms=download_ms, + poll_count=poll_count, retry_after_ms=retry_after_ms, resend_spent=resend_spent, output_bytes=output_bytes, diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index 98e3556c..2b7d8dc9 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -27,6 +27,7 @@ import json import re import time +from dataclasses import replace import httpx @@ -178,24 +179,40 @@ def submit_video( def follow_job(self, job_id: str) -> AdapterResult: """轮询已建单据 + 下载。poll GET 522/525 该次再试 1 次,不新开单。""" + poll_t0 = time.monotonic() + poll_count = 0 + + def with_poll( + result: AdapterResult, *, download_ms: int | None = None + ) -> AdapterResult: + return replace( + result, + poll_ms=int((time.monotonic() - poll_t0) * 1000), + poll_count=poll_count, + download_ms=download_ms, + ) + with self._client() as client: url = None last_status: str | None = None for _ in range(max(1, int(self._max_min * 60 // self._poll))): time.sleep(self._poll) resp = _poll_get(client, job_id) + poll_count += 1 if not (200 <= resp.status_code < 300): - return _video_http_error(resp, job_id=job_id) + return with_poll(_video_http_error(resp, job_id=job_id)) try: st = resp.json() except ValueError: - return AdapterResult( - ok=False, - error_type=ModelErrorType.INVALID_RESPONSE, - http_status=resp.status_code, - job_id=job_id, - maybe_billed=True, - edge_fingerprint="轮询响应不是 JSON", + return with_poll( + AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + job_id=job_id, + maybe_billed=True, + edge_fingerprint="轮询响应不是 JSON", + ) ) last_status = st.get("status") if last_status == "completed": @@ -203,39 +220,57 @@ def follow_job(self, job_id: str) -> AdapterResult: url = vids[0].get("url") if vids else None break if last_status in ("failed", "cancelled"): - return AdapterResult( + return with_poll( + AdapterResult( + ok=False, + error_type=ModelErrorType.UPSTREAM_FAILED, + job_id=job_id, + maybe_billed=True, + job_status=last_status, + edge_fingerprint=str(st.get("error") or ""), + ) + ) + poll_ms = int((time.monotonic() - poll_t0) * 1000) + if not url: + return replace( + AdapterResult( ok=False, - error_type=ModelErrorType.UPSTREAM_FAILED, + error_type=ModelErrorType.TIMEOUT, job_id=job_id, maybe_billed=True, - job_status=last_status, - edge_fingerprint=str(st.get("error") or ""), - ) - if not url: - return AdapterResult( - ok=False, - error_type=ModelErrorType.TIMEOUT, - job_id=job_id, - maybe_billed=True, - job_status=last_status or "timeout", + job_status=last_status or "timeout", + ), + poll_ms=poll_ms, + poll_count=poll_count, ) try: + download_t0 = time.monotonic() body = _download(client, url) + download_ms = int((time.monotonic() - download_t0) * 1000) except RuntimeError as exc: - return AdapterResult( - ok=False, - error_type=ModelErrorType.MAYBE_BILLED, + return replace( + AdapterResult( + ok=False, + error_type=ModelErrorType.MAYBE_BILLED, + job_id=job_id, + maybe_billed=True, + job_status="completed", + edge_fingerprint=str(exc), + ), + poll_ms=poll_ms, + poll_count=poll_count, + ) + return replace( + AdapterResult( + ok=True, + body=body, job_id=job_id, maybe_billed=True, job_status="completed", - edge_fingerprint=str(exc), - ) - return AdapterResult( - ok=True, - body=body, - job_id=job_id, - maybe_billed=True, - job_status="completed", + ), + poll_ms=poll_ms, + poll_count=poll_count, + download_ms=download_ms, ) def i2v( diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py index b7c83e60..0ce3ac32 100644 --- a/backend/tests/test_gateway_image.py +++ b/backend/tests/test_gateway_image.py @@ -1,3 +1,4 @@ +import json import logging import pytest @@ -99,3 +100,27 @@ def test_success_trace_has_latency_and_null_cost_by_default(caplog): gw.gen_image("p", []) assert "total_latency_ms" in caplog.text assert '"cost": null' in caplog.text or '"cost":null' in caplog.text + + +def test_skip_open_model_circuit_sets_fallback_used(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [PNG], + "gemini-2.5-flash-image-alt": [PNG], + }) + br = CircuitBreaker(cooldown_s=60) + br.open("model:gemini-2.5-flash-image") + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt", circuit=br) + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert ad.calls == ["gemini-2.5-flash-image-alt"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["fallback_used"] is True + assert line["outcome"] == "fallback_success" + assert line["route_reason"] == "skip_circuit_open" + assert not any( + r.get("outcome") == "success" and r.get("fallback_used") is False + for r in records + ) diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py index 1ccc5c29..5d58a165 100644 --- a/backend/tests/test_gateway_policy.py +++ b/backend/tests/test_gateway_policy.py @@ -1,3 +1,6 @@ +import threading +import time + from windup_common.enums.model import ModelErrorType from windup_framework.gateway.circuit import CircuitBreaker from windup_framework.gateway.policy import decide @@ -47,3 +50,34 @@ def test_circuit_opens_and_cools_down(monkeypatch): assert br.is_open("aggregator") clock["t"] = 60.0 assert not br.is_open("aggregator") + + +def test_circuit_expiry_is_thread_safe(): + clock = {"t": 0.0} + + def now() -> float: + time.sleep(0.002) + return clock["t"] + + br = CircuitBreaker(cooldown_s=60, monotonic=now) + errors: list[BaseException] = [] + + def expire_once(barrier: threading.Barrier) -> None: + try: + barrier.wait() + br.is_open("k") + except BaseException as exc: + errors.append(exc) + + clock["t"] = 0.0 + br.open("k") + clock["t"] = 60.0 + n = 8 + barrier = threading.Barrier(n) + threads = [threading.Thread(target=expire_once, args=(barrier,)) for _ in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + assert not br.is_open("k") diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py index 395cd965..b09863d6 100644 --- a/backend/tests/test_gateway_video.py +++ b/backend/tests/test_gateway_video.py @@ -1,3 +1,6 @@ +import json +import logging + import pytest from windup_common.enums.model import ModelErrorType from windup_framework.config.provider import AIProviderSettings @@ -28,12 +31,12 @@ def follow_job(self, job_id): self.followed.append(job_id) return self.follows[job_id] -def _video_gw(adapter) -> VideoGateway: +def _video_gw(adapter, circuit=None) -> VideoGateway: cfg = AIProviderSettings(video_model="kling-v2-5-turbo", video_fallbacks="kling-v2-6") return VideoGateway( registry=ModelRegistry.from_settings(cfg), adapter=adapter, - circuit=CircuitBreaker(cooldown_s=60), + circuit=circuit or CircuitBreaker(cooldown_s=60), settings=cfg, ) @@ -99,3 +102,85 @@ def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_ty _video_gw(ad).i2v(b"frame", "walk") assert ad.submit_models == ["kling-v2-5-turbo"] assert ad.followed == ["j1", "j1", "j1"] + + +def test_success_trace_has_phase_timings(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + timed = AdapterResult( + ok=True, + body=b"\x00\x00\x00\x18ftypmp42", + maybe_billed=True, + job_id="j1", + poll_count=2, + poll_ms=1500, + download_ms=80, + ) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j1": timed}, + ) + _video_gw(ad).i2v(b"frame", "walk") + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") == "success"] + assert success, caplog.text + line = success[-1] + assert line["poll_count"] == 2 + assert line["poll_ms"] == 1500 + assert line["download_ms"] == 80 + assert line["submit_ms"] is not None + + +def test_skip_open_model_circuit_sets_fallback_used(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + br = CircuitBreaker(cooldown_s=60) + br.open("model:kling-v2-5-turbo") + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j2": MP4}, + ) + body = _video_gw(ad, circuit=br).i2v(b"frame", "walk") + assert body.startswith(b"\x00\x00\x00\x18ftyp") + assert ad.submit_models == ["kling-v2-6"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["fallback_used"] is True + assert line["outcome"] == "fallback_success" + assert line["route_reason"] == "skip_circuit_open" + + +def test_submit_invalid_response_does_not_open_fallback_job(): + invalid = AdapterResult(ok=False, error_type=ModelErrorType.INVALID_RESPONSE) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [invalid, invalid, invalid], + "kling-v2-6": [AdapterResult(ok=True, job_id="j-alt", maybe_billed=True)], + }, + follows={"j-alt": MP4}, + ) + with pytest.raises(RuntimeError, match="invalid_response"): + _video_gw(ad).i2v(b"frame", "walk") + assert "kling-v2-6" not in ad.submit_models + assert ad.submit_models == ["kling-v2-5-turbo"] * 3 + + +def test_submit_ok_without_job_id_is_invalid_response(): + no_id = AdapterResult(ok=True, body=b"") + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [no_id, no_id, no_id], + "kling-v2-6": [AdapterResult(ok=True, job_id="j-alt", maybe_billed=True)], + }, + follows={"j-alt": MP4}, + ) + with pytest.raises(RuntimeError, match="invalid_response"): + _video_gw(ad).i2v(b"frame", "walk") + assert "kling-v2-6" not in ad.submit_models + assert ad.submit_models == ["kling-v2-5-turbo"] * 3 diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index a4840c15..ce62e9bb 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -722,6 +722,15 @@ def h(request): return h +def test_follow_job_records_poll_and_download_timings(): + p = _video_provider(_i2v_handler({}, statuses=("in_progress", "completed"))) + result = p.follow_job("job-1") + assert result.ok + assert result.poll_count == 2 + assert isinstance(result.poll_ms, int) + assert isinstance(result.download_ms, int) + + def test_i2v_submits_polls_and_downloads(): """一条完整的付费路径：提交拿 job id → 轮询到 completed → 下载 mp4。""" seen: dict = {} From b782d7df764794580aedb55c1236e0a8406deeb5 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:44:36 +0800 Subject: [PATCH 12/15] =?UTF-8?q?feat(gateway):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=85=A5=E5=8F=A3=E8=B7=AF=E7=94=B1=E4=B8=8E=E5=8F=B0=E8=B4=A6?= =?UTF-8?q?=E8=90=BD=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 12 + .../app/src/windup_app/bootstrap/app.py | 1 + .../src/windup_framework/config/provider.py | 27 ++ .../src/windup_framework/gateway/__init__.py | 3 + .../src/windup_framework/gateway/image.py | 313 ++++++++++------- .../src/windup_framework/gateway/ledger.py | 127 +++++++ .../src/windup_framework/gateway/models.py | 205 +++++++++++ .../src/windup_framework/gateway/routes.py | 70 ++++ .../src/windup_framework/gateway/trace.py | 12 + .../src/windup_framework/gateway/video.py | 329 ++++++++++-------- backend/tests/conftest.py | 1 + backend/tests/test_gateway_image.py | 39 +++ backend/tests/test_gateway_ledger_models.py | 61 ++++ .../tests/test_gateway_ledger_persistence.py | 73 ++++ backend/tests/test_gateway_route_config.py | 37 ++ backend/tests/test_gateway_trace.py | 4 +- backend/tests/test_gateway_video.py | 49 +++ 17 files changed, 1094 insertions(+), 269 deletions(-) create mode 100644 backend/packages/framework/src/windup_framework/gateway/ledger.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/models.py create mode 100644 backend/packages/framework/src/windup_framework/gateway/routes.py create mode 100644 backend/tests/test_gateway_ledger_models.py create mode 100644 backend/tests/test_gateway_ledger_persistence.py create mode 100644 backend/tests/test_gateway_route_config.py diff --git a/.env.example b/.env.example index 93086b4c..87e0b3d8 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,18 @@ AI_VIDEO_FALLBACKS=kling-v2-6 AI_IMAGE_UNIT_COST= AI_VIDEO_UNIT_COST_PER_SECOND= AI_PRICE_VERSION=2026-08-16 +# Gateway 路由试运行：primary 留空时复用 AI_BASE_URL / AI_API_KEY； +# fallback 三项都填才启用，用来验证 525 / SSL / 断连等 base_url 级故障切换。 +AI_ROUTE_PRIMARY_NAME=qnaigc-primary +AI_ROUTE_PRIMARY_BASE_URL= +AI_ROUTE_PRIMARY_API_KEY= +AI_ROUTE_FALLBACK_NAME= +AI_ROUTE_FALLBACK_BASE_URL= +AI_ROUTE_FALLBACK_API_KEY= +# 示例：AI_ROUTE_FALLBACK_NAME=qnaigc-backup +# 示例：AI_ROUTE_FALLBACK_BASE_URL=https://backup.example.com/v1 +# 示例：AI_ROUTE_FALLBACK_API_KEY=your-backup-ai-api-key +AI_GATEWAY_LEDGER_ENABLED=true # ── 积分定价 ── QUOTA_REGISTER_GIFT_AMOUNT=100 diff --git a/backend/packages/app/src/windup_app/bootstrap/app.py b/backend/packages/app/src/windup_app/bootstrap/app.py index 02e744ac..8080d29e 100644 --- a/backend/packages/app/src/windup_app/bootstrap/app.py +++ b/backend/packages/app/src/windup_app/bootstrap/app.py @@ -13,6 +13,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from windup_framework.db import Base, engine +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail # noqa: F401 # 模型导入：触发 Base.metadata 注册，确保 create_all 能发现所有表 from windup_app.server.character.model import Character # noqa: F401 diff --git a/backend/packages/framework/src/windup_framework/config/provider.py b/backend/packages/framework/src/windup_framework/config/provider.py index 6d077e14..ccf9ab72 100644 --- a/backend/packages/framework/src/windup_framework/config/provider.py +++ b/backend/packages/framework/src/windup_framework/config/provider.py @@ -41,6 +41,17 @@ class AIProviderSettings(BaseSettings): video_unit_cost_per_second: float | None = None price_version: str = "2026-08-16" + # ── Gateway route spike: base_url / key route candidates ──────────────── + # 第一版仍以 env 管理。primary 留空时复用上面的 AI_BASE_URL / AI_API_KEY; + # fallback 三个字段都填才表示启用一个备用入口。 + route_primary_name: str = "primary" + route_primary_base_url: str = "" + route_primary_api_key: str = "" + route_fallback_name: str = "" + route_fallback_base_url: str = "" + route_fallback_api_key: str = "" + gateway_ledger_enabled: bool = True + @field_validator("image_unit_cost", "video_unit_cost_per_second", mode="before") @classmethod def _empty_cost_is_none(cls, v): @@ -52,5 +63,21 @@ def _empty_cost_is_none(cls, v): def normalized_base_url(self) -> str: return self.base_url.rstrip("/") + @property + def effective_route_primary_base_url(self) -> str: + return (self.route_primary_base_url or self.base_url).rstrip("/") + + @property + def effective_route_primary_api_key(self) -> str: + return self.route_primary_api_key or self.api_key + + @property + def route_fallback_enabled(self) -> bool: + return all(( + self.route_fallback_name.strip(), + self.route_fallback_base_url.strip(), + self.route_fallback_api_key.strip(), + )) + settings = AIProviderSettings() diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py index 17dbf637..cdd9ce89 100644 --- a/backend/packages/framework/src/windup_framework/gateway/__init__.py +++ b/backend/packages/framework/src/windup_framework/gateway/__init__.py @@ -1,8 +1,11 @@ from windup_framework.gateway.context import bind_call_context from windup_framework.gateway.image import ImageGateway, build_image_gateway +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail from windup_framework.gateway.video import VideoGateway, build_video_gateway __all__ = [ + "AIGatewayAttempt", + "AIGatewayAttemptDetail", "ImageGateway", "VideoGateway", "bind_call_context", diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py index ad294e09..7e9ae23c 100644 --- a/backend/packages/framework/src/windup_framework/gateway/image.py +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -3,7 +3,6 @@ import time import uuid from datetime import datetime, timezone -from urllib.parse import urlparse from windup_common.enums.model import ModelErrorType from windup_framework.config.provider import AIProviderSettings, settings as default_settings @@ -11,6 +10,12 @@ from windup_framework.gateway.context import current_call_context from windup_framework.gateway.policy import decide from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + route_layer_for, + routes_from_settings, +) from windup_framework.gateway.trace import ( AttemptTrace, emit, @@ -30,21 +35,27 @@ def _utc_now() -> str: class ImageGateway: - def __init__(self, registry, adapter, circuit, settings) -> None: + def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> None: self._registry = registry self._adapter = adapter self._circuit = circuit self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHARACTER_IMAGE.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return self._route_adapters.get(route.base_url_id, self._adapter) def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: ctx = current_call_context() request_id = ctx.request_id or str(uuid.uuid4()) started = time.monotonic() input_hash = hash_image_input(prompt, refs) - host = urlparse(self._settings.base_url).hostname last_http_status: int | None = None fallback_used = False fallback_reason: str | None = None + route_reason_override: str | None = None + routes = self._routes chain = list(self._registry.chain(Scene.CHARACTER_IMAGE)) if ctx.start_from_model and ctx.start_from_model in chain: @@ -64,11 +75,12 @@ def fail(http_status: int | None) -> None: if self._circuit.is_open("aggregator"): model = models[0] if models else "" + route = routes[0] self._emit( request_id=request_id, ctx=ctx, model=model, - host=host, + route=route, attempt_index=start_i, retry_count=0, route_reason="skip_circuit_open", @@ -80,85 +92,146 @@ def fail(http_status: int | None) -> None: ) fail(None) - for i, model in enumerate(models): - attempt_index = start_i + i - if self._circuit.is_open("model:" + model): - fallback_used = True - fallback_reason = "skip" - self._emit( - request_id=request_id, - ctx=ctx, - model=model, - host=host, - attempt_index=attempt_index, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="model", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - ) - continue + for route_index, route in enumerate(routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) - if i == 0: - route_reason = ( - "start_from_caller" - if ctx.start_from_model and ctx.start_from_model in chain - else "primary" - ) - elif fallback_reason == "429": - route_reason = "fallback_after_429" - elif fallback_reason == "skip": - route_reason = "skip_circuit_open" - else: - route_reason = "fallback_after_upstream_fail" + adapter = self._adapter_for(route) + switch_to_next_route = False + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="model", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + ) + continue + + if i == 0: + route_reason = route_reason_override or ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" + ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + result = adapter.submit_image(prompt, refs, model) + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + billed = result.ok or result.maybe_billed + cost = estimate_cost( + Scene.CHARACTER_IMAGE, + billed=billed, + seconds=0, + image_unit_cost=self._settings.image_unit_cost, + video_unit_cost_per_second=self._settings.video_unit_cost_per_second, + ) + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + if result.ok: + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + circuit_scope=None, + outcome="fallback_success" if fallback_used else "success", + input_hash=input_hash, + output_hash=hash_bytes(result.body), + total_latency_ms=total_ms(), + fallback_used=fallback_used, + http_status=result.http_status, + maybe_billed=True, + cost=cost, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + output_bytes=len(result.body), + expected_bytes=result.expected_bytes, + provider_usage=result.provider_usage, + edge_fingerprint=result.edge_fingerprint or None, + job_id=result.job_id, + job_status=result.job_status, + retry_after_ms=retry_after_ms, + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=bool(result.job_id), + ) + circuit_scope = None + has_next_route = route_index + 1 < len(routes) + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" - retry_count = 0 - resend_spent = 0 - while True: - attempt_t0 = time.monotonic() - started_at = _utc_now() - result = self._adapter.submit_image(prompt, refs, model) - ended_at = _utc_now() - attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) - last_http_status = result.http_status - billed = result.ok or result.maybe_billed - cost = estimate_cost( - Scene.CHARACTER_IMAGE, - billed=billed, - seconds=0, - image_unit_cost=self._settings.image_unit_cost, - video_unit_cost_per_second=self._settings.video_unit_cost_per_second, - ) - retry_after_ms = ( - int(result.retry_after_s * 1000) - if result.retry_after_s is not None - else None - ) - if result.ok: self._emit( request_id=request_id, ctx=ctx, model=model, - host=host, + route=route, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, - circuit_scope=None, - outcome="fallback_success" if fallback_used else "success", + circuit_scope=circuit_scope, + outcome="failed", input_hash=input_hash, - output_hash=hash_bytes(result.body), + output_hash=None, total_latency_ms=total_ms(), fallback_used=fallback_used, http_status=result.http_status, - maybe_billed=True, + error_type=error_type.value, + maybe_billed=result.maybe_billed, cost=cost, started_at=started_at, ended_at=ended_at, attempt_latency_ms=attempt_latency_ms, resend_spent=resend_spent, - output_bytes=len(result.body), + output_bytes=result.output_bytes or None, expected_bytes=result.expected_bytes, provider_usage=result.provider_usage, edge_fingerprint=result.edge_fingerprint or None, @@ -166,71 +239,35 @@ def fail(http_status: int | None) -> None: job_status=result.job_status, retry_after_ms=retry_after_ms, ) - return result.body - - error_type = result.error_type or ModelErrorType.UNKNOWN - step = decide( - error_type=error_type, - retry_count=retry_count, - has_job_id=bool(result.job_id), - ) - circuit_scope = None - if step is NextStep.OPEN_AGGREGATOR: - self._circuit.open("aggregator") - circuit_scope = "aggregator" - elif step is NextStep.FALLBACK: - self._circuit.open("model:" + model) - circuit_scope = "model" - - self._emit( - request_id=request_id, - ctx=ctx, - model=model, - host=host, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - circuit_scope=circuit_scope, - outcome="failed", - input_hash=input_hash, - output_hash=None, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - http_status=result.http_status, - error_type=error_type.value, - maybe_billed=result.maybe_billed, - cost=cost, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - output_bytes=result.output_bytes or None, - expected_bytes=result.expected_bytes, - provider_usage=result.provider_usage, - edge_fingerprint=result.edge_fingerprint or None, - job_id=result.job_id, - job_status=result.job_status, - retry_after_ms=retry_after_ms, - ) - if step is NextStep.RETRY_SAME: - if error_type is ModelErrorType.RATE_LIMIT: - wait = ( - result.retry_after_s - if result.retry_after_s is not None - else _DEFAULT_RETRY_AFTER_S + if step is NextStep.OPEN_AGGREGATOR and has_next_route: + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" ) - time.sleep(min(wait, _SLEEP_CAP_S)) - retry_count += 1 - if error_type is ModelErrorType.UNREACHED: - resend_spent = 1 - continue - if step is NextStep.FALLBACK: - fallback_used = True - fallback_reason = ( - "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" - ) + break + fail(last_http_status) + if switch_to_next_route: break - fail(last_http_status) + if switch_to_next_route: + continue + route_reason_override = None fail(last_http_status) @@ -240,7 +277,7 @@ def _emit( request_id: str, ctx, model: str, - host: str | None, + route: GatewayRoute, attempt_index: int, retry_count: int, route_reason: str, @@ -278,10 +315,17 @@ def _emit( scene=Scene.CHARACTER_IMAGE, model=model, family=family, - base_url_host=host, + route_id=route.route_id, + route_group=route.route_group, + candidate_index=route.candidate_index, + provider_name=route.provider_name, + base_url_id=route.base_url_id, + base_url_host=route.host, + api_key_id=route.api_key_id, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, + route_layer=route_layer_for(route_reason), circuit_scope=circuit_scope, error_type=error_type, http_status=http_status, @@ -314,13 +358,20 @@ def _emit( def build_image_gateway(config=None, *, adapter=None, circuit=None) -> ImageGateway: cfg: AIProviderSettings = config or default_settings + route_adapters = None if adapter is None: from windup_framework.providers.sufy import SufyImageProvider - adapter = SufyImageProvider(config=cfg) + routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_IMAGE.value) + route_adapters = { + route.base_url_id: SufyImageProvider(config=config_for_route(cfg, route)) + for route in routes + } + adapter = route_adapters[routes[0].base_url_id] return ImageGateway( ModelRegistry.from_settings(cfg), adapter, circuit if circuit is not None else _CIRCUIT, cfg, + route_adapters=route_adapters, ) diff --git a/backend/packages/framework/src/windup_framework/gateway/ledger.py b/backend/packages/framework/src/windup_framework/gateway/ledger.py new file mode 100644 index 00000000..4bfb700f --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/ledger.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +from windup_framework.db import SessionLocal +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail +from windup_framework.gateway.trace import AttemptTrace + +logger = logging.getLogger("windup.gateway.ledger") + + +def _uuid(value: str | None) -> uuid.UUID: + return uuid.UUID(value) if value else uuid.uuid4() + + +def _int_or_none(value: str | None) -> int | None: + if value is None or value == "": + return None + return int(value) + + +def _dt_or_now(value: str | None) -> datetime: + if not value: + return datetime.now(timezone.utc) + return datetime.fromisoformat(value) + + +def _cost_or_none(value: float | None) -> Decimal | None: + if value is None: + return None + return Decimal(str(value)) + + +def _ledger_outcome(value: str | None) -> str: + if value == "fallback_success": + return "success" + if value in {"success", "accepted", "failed"}: + return value + return "failed" + + +def _json_or_none(value: Any) -> Any: + if value is None: + return None + if isinstance(value, dict | list | str | int | float | bool): + return value + return {"value": str(value)} + + +def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> None: + """Persist one gateway attempt without letting ledger failures affect generation.""" + + attempt_uuid = _uuid(trace.attempt_id) + try: + with session_factory() as session: + session.add( + AIGatewayAttempt( + request_id=trace.request_id, + attempt_id=attempt_uuid, + task_id=_int_or_none(trace.task_id), + user_id=_int_or_none(trace.user_id), + project_id=None, + scene=trace.scene.value, + attempt_index=trace.attempt_index or 0, + retry_count=trace.retry_count, + route_id=trace.route_id or "default.primary", + route_group=trace.route_group or trace.scene.value, + candidate_index=trace.candidate_index or 0, + provider_name=trace.provider_name or "openai-compatible", + base_url_id=trace.base_url_id or "primary", + base_url_host=trace.base_url_host or "", + api_key_id=trace.api_key_id, + model=trace.model, + family=trace.family or "", + route_reason=trace.route_reason or "primary", + route_layer=trace.route_layer or "none", + circuit_scope=trace.circuit_scope, + phase="image_sync" if trace.scene.value == "character_image" else "submit", + outcome=_ledger_outcome(trace.outcome), + job_id=trace.job_id, + error_type=trace.error_type, + http_status=trace.http_status, + maybe_billed=bool(trace.maybe_billed), + estimated_cost=_cost_or_none(trace.cost), + cost_currency="USD" if trace.cost is not None else None, + price_version=trace.price_version, + started_at=_dt_or_now(trace.started_at), + ended_at=_dt_or_now(trace.ended_at), + attempt_latency_ms=trace.attempt_latency_ms, + ) + ) + session.commit() + except Exception: + logger.exception("Gateway hot ledger write failed request_id=%s", trace.request_id) + return + + try: + with session_factory() as session: + session.add( + AIGatewayAttemptDetail( + attempt_id=attempt_uuid, + request_id=trace.request_id, + task_id=_int_or_none(trace.task_id), + job_status=trace.job_status, + edge_fingerprint=trace.edge_fingerprint, + error_message=None, + provider_request_id=None, + provider_usage=_json_or_none(trace.provider_usage), + input_hash=trace.input_hash, + output_hash=trace.output_hash, + output_bytes=trace.output_bytes, + expected_bytes=trace.expected_bytes, + retry_after_ms=trace.retry_after_ms, + submit_ms=trace.submit_ms, + poll_ms=trace.poll_ms, + download_ms=trace.download_ms, + poll_count=trace.poll_count, + extra=None, + ) + ) + session.commit() + except Exception: + logger.exception("Gateway detail ledger write failed request_id=%s", trace.request_id) diff --git a/backend/packages/framework/src/windup_framework/gateway/models.py b/backend/packages/framework/src/windup_framework/gateway/models.py new file mode 100644 index 00000000..95cf95d7 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/models.py @@ -0,0 +1,205 @@ +"""Gateway attempt ledger ORM models. + +The hot/cold split keeps route health and cost attribution queries on a compact +table, while larger troubleshooting payloads live in the detail table. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + Boolean, + CheckConstraint, + DateTime, + Index, + Integer, + JSON, + Numeric, + String, + Text, + Uuid, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from windup_framework.db import Base + + +_BIGINT = BigInteger().with_variant(Integer, "sqlite") +_JSONB = JSON().with_variant(JSONB, "postgresql") + + +class AIGatewayAttempt(Base): + """Hot ledger row for one gateway attempt. + + One row represents one actual attempt against a model/key/base_url candidate. + It intentionally stores only compact routing, outcome, and cost fields. + """ + + __tablename__ = "windup_ai_gateway_attempt" + __table_args__ = ( + CheckConstraint( + "scene IN ('character_image', 'character_action')", + name="ck_gateway_attempt_scene", + ), + CheckConstraint( + "phase IN ('image_sync', 'submit', 'follow', 'download')", + name="ck_gateway_attempt_phase", + ), + CheckConstraint( + "route_layer IN ('none', 'model', 'key', 'base_url')", + name="ck_gateway_attempt_route_layer", + ), + CheckConstraint( + "outcome IN ('success', 'accepted', 'failed')", + name="ck_gateway_attempt_outcome", + ), + CheckConstraint( + "http_status IS NULL OR (http_status >= 100 AND http_status <= 599)", + name="ck_gateway_attempt_http_status", + ), + CheckConstraint( + "estimated_cost IS NULL OR estimated_cost >= 0", + name="ck_gateway_attempt_cost_non_negative", + ), + CheckConstraint( + "attempt_index >= 0 " + "AND retry_count >= 0 " + "AND candidate_index >= 0 " + "AND (attempt_latency_ms IS NULL OR attempt_latency_ms >= 0)", + name="ck_gateway_attempt_non_negative_counts", + ), + Index("ix_gateway_attempt_request", "request_id", "attempt_index"), + Index("ix_gateway_attempt_task", "task_id", "scene", "created_at"), + Index( + "ix_gateway_attempt_provider_error", + "provider_name", + "base_url_id", + "error_type", + "created_at", + ), + Index( + "ix_gateway_attempt_key_error", + "provider_name", + "base_url_id", + "api_key_id", + "error_type", + "created_at", + ), + Index("ix_gateway_attempt_model_error", "model", "error_type", "created_at"), + Index( + "ix_gateway_attempt_route_health", + "route_group", + "route_id", + "outcome", + "created_at", + ), + Index("ix_gateway_attempt_maybe_billed", "maybe_billed", "outcome", "created_at"), + Index("ix_gateway_attempt_job", "job_id"), + ) + + id: Mapped[int] = mapped_column(_BIGINT, primary_key=True, autoincrement=True) + request_id: Mapped[str] = mapped_column(String(96), nullable=False) + attempt_id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), unique=True, nullable=False) + + task_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + user_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + project_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + scene: Mapped[str] = mapped_column(Text, nullable=False) + + attempt_index: Mapped[int] = mapped_column(Integer, nullable=False) + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + route_id: Mapped[str] = mapped_column(Text, nullable=False) + route_group: Mapped[str] = mapped_column(Text, nullable=False) + candidate_index: Mapped[int] = mapped_column(Integer, nullable=False) + + provider_name: Mapped[str] = mapped_column(Text, nullable=False) + base_url_id: Mapped[str] = mapped_column(Text, nullable=False) + base_url_host: Mapped[str] = mapped_column(Text, nullable=False) + api_key_id: Mapped[str | None] = mapped_column(Text, nullable=True) + + model: Mapped[str] = mapped_column(Text, nullable=False) + family: Mapped[str] = mapped_column(Text, nullable=False) + + route_reason: Mapped[str] = mapped_column(Text, nullable=False) + route_layer: Mapped[str] = mapped_column(Text, nullable=False, default="none") + circuit_scope: Mapped[str | None] = mapped_column(Text, nullable=True) + phase: Mapped[str] = mapped_column(Text, nullable=False) + outcome: Mapped[str] = mapped_column(Text, nullable=False) + + job_id: Mapped[str | None] = mapped_column(Text, nullable=True) + + error_type: Mapped[str | None] = mapped_column(Text, nullable=True) + http_status: Mapped[int | None] = mapped_column(Integer, nullable=True) + + maybe_billed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + estimated_cost: Mapped[Decimal | None] = mapped_column(Numeric(14, 6), nullable=True) + cost_currency: Mapped[str | None] = mapped_column(String(8), nullable=True) + price_version: Mapped[str | None] = mapped_column(Text, nullable=True) + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + attempt_latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class AIGatewayAttemptDetail(Base): + """Cold troubleshooting row for one gateway attempt.""" + + __tablename__ = "windup_ai_gateway_attempt_detail" + __table_args__ = ( + CheckConstraint( + "(output_bytes IS NULL OR output_bytes >= 0) " + "AND (expected_bytes IS NULL OR expected_bytes >= 0) " + "AND (retry_after_ms IS NULL OR retry_after_ms >= 0) " + "AND (submit_ms IS NULL OR submit_ms >= 0) " + "AND (poll_ms IS NULL OR poll_ms >= 0) " + "AND (download_ms IS NULL OR download_ms >= 0) " + "AND (poll_count IS NULL OR poll_count >= 0)", + name="ck_gateway_attempt_detail_non_negative_counts", + ), + Index("ix_gateway_attempt_detail_request", "request_id"), + Index("ix_gateway_attempt_detail_task", "task_id", "created_at"), + Index("ix_gateway_attempt_detail_job_status", "job_status", "created_at"), + ) + + id: Mapped[int] = mapped_column(_BIGINT, primary_key=True, autoincrement=True) + attempt_id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), unique=True, nullable=False) + request_id: Mapped[str] = mapped_column(String(96), nullable=False) + task_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + + job_status: Mapped[str | None] = mapped_column(Text, nullable=True) + edge_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + provider_request_id: Mapped[str | None] = mapped_column(Text, nullable=True) + provider_usage: Mapped[dict | None] = mapped_column(_JSONB, nullable=True) + + input_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + output_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + output_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + expected_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + + retry_after_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + submit_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + poll_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + download_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + poll_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + extra: Mapped[dict | None] = mapped_column(_JSONB, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/routes.py b/backend/packages/framework/src/windup_framework/gateway/routes.py new file mode 100644 index 00000000..7438ada2 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/routes.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse + +from windup_framework.config.provider import AIProviderSettings + + +@dataclass(frozen=True) +class GatewayRoute: + route_id: str + route_group: str + candidate_index: int + provider_name: str + base_url_id: str + base_url: str + api_key_id: str | None + api_key: str + + @property + def host(self) -> str | None: + return urlparse(self.base_url).hostname + + +def routes_from_settings(cfg: AIProviderSettings, *, route_group: str) -> tuple[GatewayRoute, ...]: + primary_name = cfg.route_primary_name.strip() or "primary" + routes = [ + GatewayRoute( + route_id=f"{primary_name}.primary", + route_group=route_group, + candidate_index=0, + provider_name=cfg.provider, + base_url_id=primary_name, + base_url=cfg.effective_route_primary_base_url, + api_key_id=primary_name, + api_key=cfg.effective_route_primary_api_key, + ) + ] + if cfg.route_fallback_enabled: + fallback_name = cfg.route_fallback_name.strip() + routes.append( + GatewayRoute( + route_id=f"{fallback_name}.fallback", + route_group=route_group, + candidate_index=1, + provider_name=cfg.provider, + base_url_id=fallback_name, + base_url=cfg.route_fallback_base_url.rstrip("/"), + api_key_id=fallback_name, + api_key=cfg.route_fallback_api_key, + ) + ) + return tuple(routes) + + +def config_for_route(cfg: AIProviderSettings, route: GatewayRoute) -> AIProviderSettings: + return cfg.model_copy(update={"base_url": route.base_url, "api_key": route.api_key}) + + +def route_layer_for(reason: str) -> str: + if reason == "base_url_unreached": + return "base_url" + if reason in { + "fallback_after_429", + "fallback_after_upstream_fail", + "skip_circuit_open", + "start_from_caller", + }: + return "model" + return "none" diff --git a/backend/packages/framework/src/windup_framework/gateway/trace.py b/backend/packages/framework/src/windup_framework/gateway/trace.py index c828cc16..4a37050c 100644 --- a/backend/packages/framework/src/windup_framework/gateway/trace.py +++ b/backend/packages/framework/src/windup_framework/gateway/trace.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, fields from enum import Enum +from windup_framework.config.provider import settings as provider_settings from windup_framework.gateway.types import Scene logger = logging.getLogger("windup.gateway") @@ -20,10 +21,17 @@ class AttemptTrace: task_id: str | None = None user_id: str | None = None family: str | None = None + route_id: str | None = None + route_group: str | None = None + candidate_index: int | None = None + provider_name: str | None = None + base_url_id: str | None = None base_url_host: str | None = None + api_key_id: str | None = None attempt_index: int | None = None retry_count: int = 0 route_reason: str | None = None + route_layer: str | None = None circuit_scope: str | None = None error_type: str | None = None http_status: int | None = None @@ -91,3 +99,7 @@ def hash_bytes(data: bytes) -> str: def emit(trace: AttemptTrace) -> None: logger.info("%s", json.dumps(trace.as_dict(), ensure_ascii=False, default=str)) + if provider_settings.gateway_ledger_enabled: + from windup_framework.gateway.ledger import persist_attempt + + persist_attempt(trace) diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index e45832e6..ff82c683 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -4,7 +4,6 @@ import uuid from dataclasses import replace from datetime import datetime, timezone -from urllib.parse import urlparse from windup_common.enums.model import ModelErrorType from windup_framework.config.provider import AIProviderSettings, settings as default_settings @@ -12,6 +11,12 @@ from windup_framework.gateway.image import _CIRCUIT from windup_framework.gateway.policy import decide from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + route_layer_for, + routes_from_settings, +) from windup_framework.gateway.trace import ( AttemptTrace, emit, @@ -30,11 +35,16 @@ def _utc_now() -> str: class VideoGateway: - def __init__(self, registry, adapter, circuit, settings) -> None: + def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> None: self._registry = registry self._adapter = adapter self._circuit = circuit self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHARACTER_ACTION.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return self._route_adapters.get(route.base_url_id, self._adapter) def i2v( self, @@ -47,11 +57,12 @@ def i2v( request_id = ctx.request_id or str(uuid.uuid4()) started = time.monotonic() input_hash = hash_image_input(prompt, [first_frame]) - host = urlparse(self._settings.base_url).hostname last_http_status: int | None = None last_error: ModelErrorType | None = None fallback_used = False fallback_reason: str | None = None + route_reason_override: str | None = None + routes = self._routes chain = list(self._registry.chain(Scene.CHARACTER_ACTION)) if ctx.start_from_model and ctx.start_from_model in chain: @@ -73,11 +84,12 @@ def fail(http_status: int | None) -> None: if self._circuit.is_open("aggregator"): model = models[0] if models else "" + route = routes[0] self._emit( request_id=request_id, ctx=ctx, model=model, - host=host, + route=route, attempt_index=start_i, retry_count=0, route_reason="skip_circuit_open", @@ -89,75 +101,128 @@ def fail(http_status: int | None) -> None: ) fail(None) - for i, model in enumerate(models): - attempt_index = start_i + i - if self._circuit.is_open("model:" + model): - fallback_used = True - fallback_reason = "skip" - self._emit( - request_id=request_id, - ctx=ctx, - model=model, - host=host, - attempt_index=attempt_index, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="model", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - ) - continue + for route_index, route in enumerate(routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) - if i == 0: - route_reason = ( - "start_from_caller" - if ctx.start_from_model and ctx.start_from_model in chain - else "primary" - ) - elif fallback_reason == "429": - route_reason = "fallback_after_429" - elif fallback_reason == "skip": - route_reason = "skip_circuit_open" - else: - route_reason = "fallback_after_upstream_fail" + adapter = self._adapter_for(route) + switch_to_next_route = False + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" + self._emit( + request_id=request_id, + ctx=ctx, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + circuit_scope="model", + outcome="failed", + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + ) + continue - retry_count = 0 - resend_spent = 0 - bound_job_id: str | None = None - while True: - attempt_t0 = time.monotonic() - started_at = _utc_now() - submit_ms: int | None = None - if bound_job_id is None: - submit_t0 = time.monotonic() - result = self._adapter.submit_video( - first_frame, prompt, seconds, size, model + if i == 0: + route_reason = route_reason_override or ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" ) - submit_ms = int((time.monotonic() - submit_t0) * 1000) - if result.ok and result.job_id: - bound_job_id = result.job_id - result = self._adapter.follow_job(bound_job_id) - elif result.ok: - result = replace( - result, - ok=False, - error_type=ModelErrorType.INVALID_RESPONSE, - body=b"", - ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" else: - result = self._adapter.follow_job(bound_job_id) + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + bound_job_id: str | None = None + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + submit_ms: int | None = None + if bound_job_id is None: + submit_t0 = time.monotonic() + result = adapter.submit_video( + first_frame, prompt, seconds, size, model + ) + submit_ms = int((time.monotonic() - submit_t0) * 1000) + if result.ok and result.job_id: + bound_job_id = result.job_id + result = adapter.follow_job(bound_job_id) + elif result.ok: + result = replace( + result, + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + body=b"", + ) + else: + result = adapter.follow_job(bound_job_id) + + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + if result.ok: + self._emit_result( + request_id=request_id, + ctx=ctx, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="fallback_success" if fallback_used else "success", + submit_ms=submit_ms, + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + last_error = error_type + has_job_id = bool(result.job_id or bound_job_id) + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=has_job_id, + ) + circuit_scope = None + has_next_route = route_index + 1 < len(routes) + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" - ended_at = _utc_now() - attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) - last_http_status = result.http_status - if result.ok: self._emit_result( request_id=request_id, ctx=ctx, model=model, - host=host, + route=route, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, @@ -170,79 +235,55 @@ def fail(http_status: int | None) -> None: attempt_latency_ms=attempt_latency_ms, resend_spent=resend_spent, seconds=seconds, - outcome="fallback_success" if fallback_used else "success", + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type, submit_ms=submit_ms, ) - return result.body - - error_type = result.error_type or ModelErrorType.UNKNOWN - last_error = error_type - has_job_id = bool(result.job_id or bound_job_id) - step = decide( - error_type=error_type, - retry_count=retry_count, - has_job_id=has_job_id, - ) - circuit_scope = None - if step is NextStep.OPEN_AGGREGATOR: - self._circuit.open("aggregator") - circuit_scope = "aggregator" - elif step is NextStep.FALLBACK: - self._circuit.open("model:" + model) - circuit_scope = "model" - - self._emit_result( - request_id=request_id, - ctx=ctx, - model=model, - host=host, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - result=result, - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - seconds=seconds, - outcome="failed", - circuit_scope=circuit_scope, - error_type=error_type, - submit_ms=submit_ms, - ) - if step is NextStep.RETRY_SAME: - if error_type is ModelErrorType.RATE_LIMIT: - wait = ( - result.retry_after_s - if result.retry_after_s is not None - else _DEFAULT_RETRY_AFTER_S - ) - time.sleep(min(wait, _SLEEP_CAP_S)) - retry_count += 1 - if error_type is ModelErrorType.UNREACHED: - resend_spent = 1 - continue - if step is NextStep.FALLBACK: - if ( - bound_job_id is not None - and error_type is not ModelErrorType.UPSTREAM_FAILED - ): - fail(last_http_status) if ( - bound_job_id is None - and error_type is not ModelErrorType.RATE_LIMIT + step is NextStep.OPEN_AGGREGATOR + and has_next_route + and bound_job_id is None ): - fail(last_http_status) - fallback_used = True - fallback_reason = ( - "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" - ) - bound_job_id = None + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + if ( + bound_job_id is not None + and error_type is not ModelErrorType.UPSTREAM_FAILED + ): + fail(last_http_status) + if ( + bound_job_id is None + and error_type is not ModelErrorType.RATE_LIMIT + ): + fail(last_http_status) + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + bound_job_id = None + break + fail(last_http_status) + if switch_to_next_route: break - fail(last_http_status) + if switch_to_next_route: + continue + route_reason_override = None fail(last_http_status) @@ -252,7 +293,7 @@ def _emit_result( request_id: str, ctx, model: str, - host: str | None, + route: GatewayRoute, attempt_index: int, retry_count: int, route_reason: str, @@ -287,7 +328,7 @@ def _emit_result( request_id=request_id, ctx=ctx, model=model, - host=host, + route=route, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, @@ -324,7 +365,7 @@ def _emit( request_id: str, ctx, model: str, - host: str | None, + route: GatewayRoute, attempt_index: int, retry_count: int, route_reason: str, @@ -366,10 +407,17 @@ def _emit( scene=Scene.CHARACTER_ACTION, model=model, family=family, - base_url_host=host, + route_id=route.route_id, + route_group=route.route_group, + candidate_index=route.candidate_index, + provider_name=route.provider_name, + base_url_id=route.base_url_id, + base_url_host=route.host, + api_key_id=route.api_key_id, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, + route_layer=route_layer_for(route_reason), circuit_scope=circuit_scope, error_type=error_type, http_status=http_status, @@ -402,13 +450,20 @@ def _emit( def build_video_gateway(config=None, *, adapter=None, circuit=None) -> VideoGateway: cfg: AIProviderSettings = config or default_settings + route_adapters = None if adapter is None: from windup_framework.providers.sufy import SufyVideoProvider - adapter = SufyVideoProvider(config=cfg) + routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_ACTION.value) + route_adapters = { + route.base_url_id: SufyVideoProvider(config=config_for_route(cfg, route)) + for route in routes + } + adapter = route_adapters[routes[0].base_url_id] return VideoGateway( ModelRegistry.from_settings(cfg), adapter, circuit if circuit is not None else _CIRCUIT, cfg, + route_adapters=route_adapters, ) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index d8eafbda..d6a1fceb 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -12,6 +12,7 @@ # setdefault 不覆盖已有的环境变量(本地 .env 或 CI secrets 优先生效)。 os.environ.setdefault("JWT_SECRET", "test-secret-key-for-ci-only-32chars") os.environ.setdefault("POSTGRES_PASSWORD", "testpassword123") +os.environ.setdefault("AI_GATEWAY_LEDGER_ENABLED", "false") import pytest from fastapi.testclient import TestClient diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py index 0ce3ac32..9a84811c 100644 --- a/backend/tests/test_gateway_image.py +++ b/backend/tests/test_gateway_image.py @@ -47,6 +47,45 @@ def test_522_retries_same_model_once_and_does_not_fallback(): assert ad.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] +def test_522_switches_base_url_route_before_model_fallback(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + backup = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + primary, + CircuitBreaker(), + cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert primary.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + assert backup.calls == ["gemini-2.5-flash-image"] + assert "gemini-2.5-flash-image-alt" not in primary.calls + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + def test_aggregator_circuit_skips_fallback_model(): ad = FakeImageAdapter({ "gemini-2.5-flash-image": [UNREACHED, UNREACHED], diff --git a/backend/tests/test_gateway_ledger_models.py b/backend/tests/test_gateway_ledger_models.py new file mode 100644 index 00000000..be52e124 --- /dev/null +++ b/backend/tests/test_gateway_ledger_models.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from sqlalchemy import create_engine, inspect + +from windup_framework.db import Base +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail + + +def test_gateway_ledger_tables_are_registered_and_split_hot_detail(): + assert AIGatewayAttempt.__tablename__ == "windup_ai_gateway_attempt" + assert AIGatewayAttemptDetail.__tablename__ == "windup_ai_gateway_attempt_detail" + + attempt_cols = set(AIGatewayAttempt.__table__.columns.keys()) + detail_cols = set(AIGatewayAttemptDetail.__table__.columns.keys()) + + # Hot table: compact router/cost fields used by key/url/model health queries. + assert { + "request_id", + "attempt_id", + "route_id", + "route_group", + "candidate_index", + "provider_name", + "base_url_id", + "base_url_host", + "api_key_id", + "model", + "route_layer", + "error_type", + "maybe_billed", + "estimated_cost", + } <= attempt_cols + + # Cold table: larger troubleshooting fields stay out of the hot path. + assert { + "attempt_id", + "edge_fingerprint", + "error_message", + "provider_request_id", + "provider_usage", + "submit_ms", + "poll_ms", + "download_ms", + "extra", + } <= detail_cols + assert "provider_usage" not in attempt_cols + assert "edge_fingerprint" not in attempt_cols + + +def test_gateway_ledger_tables_can_be_created_in_test_db(): + engine = create_engine("sqlite:///:memory:") + try: + Base.metadata.create_all( + engine, + tables=[AIGatewayAttempt.__table__, AIGatewayAttemptDetail.__table__], + ) + tables = set(inspect(engine).get_table_names()) + assert "windup_ai_gateway_attempt" in tables + assert "windup_ai_gateway_attempt_detail" in tables + finally: + engine.dispose() diff --git a/backend/tests/test_gateway_ledger_persistence.py b/backend/tests/test_gateway_ledger_persistence.py new file mode 100644 index 00000000..3b995e09 --- /dev/null +++ b/backend/tests/test_gateway_ledger_persistence.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from windup_framework.db import Base +from windup_framework.gateway.ledger import persist_attempt +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail +from windup_framework.gateway.trace import AttemptTrace +from windup_framework.gateway.types import Scene + + +def test_persist_attempt_splits_hot_and_detail_fields(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[AIGatewayAttempt.__table__, AIGatewayAttemptDetail.__table__], + ) + session_factory = sessionmaker(bind=engine, expire_on_commit=False) + attempt_id = str(uuid.uuid4()) + + persist_attempt( + AttemptTrace( + request_id="gw-1", + attempt_id=attempt_id, + task_id="42", + user_id="7", + scene=Scene.CHARACTER_IMAGE, + model="gemini-2.5-flash-image", + family="image.chat_data_uri", + route_id="backup.fallback", + route_group="character_image", + candidate_index=1, + provider_name="openai-compatible", + base_url_id="backup", + base_url_host="backup.example.com", + api_key_id="backup", + attempt_index=2, + retry_count=1, + route_reason="base_url_unreached", + route_layer="base_url", + circuit_scope=None, + outcome="fallback_success", + edge_fingerprint="cf-ray=abc", + maybe_billed=True, + cost=0.25, + price_version="2026-08-16", + provider_usage={"total_tokens": 12}, + started_at=datetime.now(timezone.utc).isoformat(), + ended_at=datetime.now(timezone.utc).isoformat(), + attempt_latency_ms=123, + ), + session_factory=session_factory, + ) + + with session_factory() as session: + hot = session.scalar(select(AIGatewayAttempt)) + detail = session.scalar(select(AIGatewayAttemptDetail)) + + assert hot is not None + assert detail is not None + assert hot.request_id == "gw-1" + assert hot.task_id == 42 + assert hot.user_id == 7 + assert hot.base_url_id == "backup" + assert hot.route_layer == "base_url" + assert hot.outcome == "success" + assert str(hot.attempt_id) == attempt_id + assert detail.edge_fingerprint == "cf-ray=abc" + assert detail.provider_usage == {"total_tokens": 12} diff --git a/backend/tests/test_gateway_route_config.py b/backend/tests/test_gateway_route_config.py new file mode 100644 index 00000000..ad4ffef1 --- /dev/null +++ b/backend/tests/test_gateway_route_config.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from windup_framework.config.provider import AIProviderSettings + + +def test_gateway_route_env_fields_are_live(): + cfg = AIProviderSettings( + route_primary_name="qnaigc", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + + assert cfg.route_primary_name == "qnaigc" + assert cfg.route_primary_base_url == "https://api.qnaigc.com/v1" + assert cfg.route_primary_api_key == "primary-key" + assert cfg.route_fallback_name == "backup" + assert cfg.route_fallback_base_url == "https://backup.example.com/v1" + assert cfg.route_fallback_api_key == "backup-key" + + +def test_empty_gateway_route_values_disable_fallback_route(): + cfg = AIProviderSettings( + base_url="https://api.qnaigc.com/v1/", + api_key="legacy-key", + route_primary_base_url="", + route_primary_api_key="", + route_fallback_name="", + route_fallback_base_url="", + route_fallback_api_key="", + ) + + assert cfg.effective_route_primary_base_url == "https://api.qnaigc.com/v1" + assert cfg.effective_route_primary_api_key == "legacy-key" + assert cfg.route_fallback_enabled is False diff --git a/backend/tests/test_gateway_trace.py b/backend/tests/test_gateway_trace.py index e022b1dc..6bf04e44 100644 --- a/backend/tests/test_gateway_trace.py +++ b/backend/tests/test_gateway_trace.py @@ -6,7 +6,9 @@ REQUIRED = { "request_id", "attempt_id", "task_id", "user_id", "scene", "model", "family", - "base_url_host", "attempt_index", "retry_count", "route_reason", "circuit_scope", + "route_id", "route_group", "candidate_index", "provider_name", "base_url_id", + "base_url_host", "api_key_id", "attempt_index", "retry_count", "route_reason", + "route_layer", "circuit_scope", "error_type", "http_status", "edge_fingerprint", "job_id", "fallback_used", "outcome", "job_status", "started_at", "ended_at", "attempt_latency_ms", "total_latency_ms", "submit_ms", "poll_ms", "download_ms", "poll_count", diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py index b09863d6..ebef5fc0 100644 --- a/backend/tests/test_gateway_video.py +++ b/backend/tests/test_gateway_video.py @@ -52,6 +52,55 @@ def test_submit_522_retries_once_does_not_open_second_job_on_fallback_model(): assert ad.submit_models == ["kling-v2-5-turbo", "kling-v2-5-turbo"] assert ad.followed == [] + +def test_submit_522_switches_base_url_route_before_model_fallback(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [UNREACHED, UNREACHED], + "kling-v2-6": [AdapterResult(ok=True, job_id="wrong", maybe_billed=True)], + }, + follows={}, + ) + backup = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j-backup", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j-backup": MP4}, + ) + cfg = AIProviderSettings( + video_model="kling-v2-5-turbo", + video_fallbacks="kling-v2-6", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=primary, + circuit=CircuitBreaker(cooldown_s=60), + settings=cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.i2v(b"frame", "walk").startswith(b"\x00\x00\x00\x18ftyp") + assert primary.submit_models == ["kling-v2-5-turbo", "kling-v2-5-turbo"] + assert backup.submit_models == ["kling-v2-5-turbo"] + assert "kling-v2-6" not in primary.submit_models + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + def test_follow_failed_opens_new_job_on_fallback(): ad = FakeVideoAdapter( submits={ From c47a0574e2d32eb7f732d7e1ffe326b4025379e9 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:46:44 +0800 Subject: [PATCH 13/15] =?UTF-8?q?feat(gateway):=20=E5=90=8C=E5=85=A5?= =?UTF-8?q?=E5=8F=A3=E5=A4=9A=20key=20=E5=88=87=E6=8D=A2=E5=B9=B6=E6=8B=86?= =?UTF-8?q?=E5=88=86=20attempt=20=E5=86=B7=E7=83=AD=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 429 换 key 不换模型，522 跳过该入口剩余 key；Chat 走同一套路由。 AttemptTrace 只保留热字段，排障详情进 AttemptDetail。 --- .env.example | 5 + .../src/windup_framework/config/provider.py | 4 + .../src/windup_framework/gateway/__init__.py | 3 + .../src/windup_framework/gateway/chat.py | 336 ++++++++++++++++++ .../src/windup_framework/gateway/image.py | 291 +++++++-------- .../src/windup_framework/gateway/ledger.py | 55 +-- .../src/windup_framework/gateway/models.py | 4 +- .../src/windup_framework/gateway/policy.py | 2 +- .../src/windup_framework/gateway/routes.py | 87 ++++- .../src/windup_framework/gateway/trace.py | 73 ++-- .../src/windup_framework/gateway/types.py | 3 + .../src/windup_framework/gateway/video.py | 228 +++++------- .../src/windup_framework/providers/chat.py | 20 +- backend/tests/test_gateway_chat.py | 95 +++++ backend/tests/test_gateway_image.py | 80 ++++- .../tests/test_gateway_ledger_persistence.py | 28 +- backend/tests/test_gateway_policy.py | 4 +- backend/tests/test_gateway_route_config.py | 24 ++ backend/tests/test_gateway_trace.py | 91 ++++- backend/tests/test_gateway_video.py | 39 ++ 20 files changed, 1063 insertions(+), 409 deletions(-) create mode 100644 backend/packages/framework/src/windup_framework/gateway/chat.py create mode 100644 backend/tests/test_gateway_chat.py diff --git a/.env.example b/.env.example index 87e0b3d8..131cdd35 100644 --- a/.env.example +++ b/.env.example @@ -33,8 +33,10 @@ QINIU_PRIVATE_SPACE=false # 键名前缀必须是 AI_，由 framework/config/provider.py 的 env_prefix 决定。 AI_BASE_URL=https://api.qnaigc.com/v1 AI_API_KEY=your-ai-api-key +AI_MODEL=your-chat-model # 各能力分开配型号：三条能力同时在用不同模型，共用一个字段会换一个连带换全部。 # 取值即默认值——不写这两行时跑的就是它们。 +AI_CHAT_FALLBACKS= AI_IMAGE_MODEL=gemini-2.5-flash-image AI_VIDEO_MODEL=kling-v2-5-turbo AI_IMAGE_FALLBACKS= @@ -44,12 +46,15 @@ AI_VIDEO_UNIT_COST_PER_SECOND= AI_PRICE_VERSION=2026-08-16 # Gateway 路由试运行：primary 留空时复用 AI_BASE_URL / AI_API_KEY； # fallback 三项都填才启用，用来验证 525 / SSL / 断连等 base_url 级故障切换。 +# 同入口多个 key 用逗号分隔：429 换下一个 key，522/525 跳过该入口剩余 key。 AI_ROUTE_PRIMARY_NAME=qnaigc-primary AI_ROUTE_PRIMARY_BASE_URL= AI_ROUTE_PRIMARY_API_KEY= +AI_ROUTE_PRIMARY_API_KEYS= AI_ROUTE_FALLBACK_NAME= AI_ROUTE_FALLBACK_BASE_URL= AI_ROUTE_FALLBACK_API_KEY= +AI_ROUTE_FALLBACK_API_KEYS= # 示例：AI_ROUTE_FALLBACK_NAME=qnaigc-backup # 示例：AI_ROUTE_FALLBACK_BASE_URL=https://backup.example.com/v1 # 示例：AI_ROUTE_FALLBACK_API_KEY=your-backup-ai-api-key diff --git a/backend/packages/framework/src/windup_framework/config/provider.py b/backend/packages/framework/src/windup_framework/config/provider.py index ccf9ab72..efbe79b5 100644 --- a/backend/packages/framework/src/windup_framework/config/provider.py +++ b/backend/packages/framework/src/windup_framework/config/provider.py @@ -35,6 +35,7 @@ class AIProviderSettings(BaseSettings): video_model: str = "kling-v2-5-turbo" image_model: str = "gemini-2.5-flash-image" + chat_fallbacks: str = "" image_fallbacks: str = "" video_fallbacks: str = "" image_unit_cost: float | None = None @@ -44,12 +45,15 @@ class AIProviderSettings(BaseSettings): # ── Gateway route spike: base_url / key route candidates ──────────────── # 第一版仍以 env 管理。primary 留空时复用上面的 AI_BASE_URL / AI_API_KEY; # fallback 三个字段都填才表示启用一个备用入口。 + # *_API_KEYS 是同入口额外 key（逗号分隔）：429 换 key，UNREACHED 跳过剩余 key。 route_primary_name: str = "primary" route_primary_base_url: str = "" route_primary_api_key: str = "" + route_primary_api_keys: str = "" route_fallback_name: str = "" route_fallback_base_url: str = "" route_fallback_api_key: str = "" + route_fallback_api_keys: str = "" gateway_ledger_enabled: bool = True @field_validator("image_unit_cost", "video_unit_cost_per_second", mode="before") diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py index cdd9ce89..da0396e9 100644 --- a/backend/packages/framework/src/windup_framework/gateway/__init__.py +++ b/backend/packages/framework/src/windup_framework/gateway/__init__.py @@ -1,3 +1,4 @@ +from windup_framework.gateway.chat import ChatGateway, build_chat_gateway from windup_framework.gateway.context import bind_call_context from windup_framework.gateway.image import ImageGateway, build_image_gateway from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail @@ -6,9 +7,11 @@ __all__ = [ "AIGatewayAttempt", "AIGatewayAttemptDetail", + "ChatGateway", "ImageGateway", "VideoGateway", "bind_call_context", + "build_chat_gateway", "build_image_gateway", "build_video_gateway", ] diff --git a/backend/packages/framework/src/windup_framework/gateway/chat.py b/backend/packages/framework/src/windup_framework/gateway/chat.py new file mode 100644 index 00000000..ccb95888 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/chat.py @@ -0,0 +1,336 @@ +from __future__ import annotations + +import hashlib +import json +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +import httpx +from langchain_openai import ChatOpenAI + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.policy import decide +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + key_circuit_id, + lookup_adapter, + routes_from_settings, +) +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace, emit +from windup_framework.gateway.types import Family, NextStep, Scene + +_CIRCUIT = CircuitBreaker() +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +@dataclass(frozen=True) +class ChatAdapterResult: + ok: bool + value: Any = None + error_type: ModelErrorType | None = None + http_status: int | None = None + edge_fingerprint: str = "" + retry_after_s: float | None = None + provider_usage: object | None = None + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_fallbacks(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +def _hash_messages(messages: Any) -> str: + payload = json.dumps(messages, ensure_ascii=False, default=str, sort_keys=True) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _error_type_from_exception(exc: Exception) -> tuple[ModelErrorType, int | None, str]: + status = getattr(exc, "status_code", None) + response = getattr(exc, "response", None) + if status is None and response is not None: + status = getattr(response, "status_code", None) + if isinstance(status, int): + from windup_framework.gateway.classify import classify_http + + return classify_http(status), status, str(exc)[:200] + if isinstance(exc, (httpx.ConnectError, httpx.NetworkError)): + return ModelErrorType.UNREACHED, None, str(exc)[:200] + if isinstance(exc, (httpx.ReadTimeout, httpx.TimeoutException, TimeoutError)): + return ModelErrorType.TIMEOUT, None, str(exc)[:200] + return ModelErrorType.UNKNOWN, None, str(exc)[:200] + + +class LangChainChatAdapter: + """Protocol adapter: Gateway policy around LangChain's ChatOpenAI client.""" + + def __init__(self, config: AIProviderSettings, **client_kwargs: Any) -> None: + self._cfg = config + self._client_kwargs = client_kwargs + + def invoke(self, messages: Any, *, model: str, **kwargs: Any) -> ChatAdapterResult: + client = ChatOpenAI( + model=model, + api_key=self._cfg.api_key or None, + base_url=self._cfg.normalized_base_url, + timeout=self._cfg.timeout, + # Gateway owns retry/circuit accounting; hidden SDK retries blur attempts. + max_retries=0, + **self._client_kwargs, + ) + try: + return ChatAdapterResult(ok=True, value=client.invoke(messages, **kwargs)) + except Exception as exc: + error_type, status, edge = _error_type_from_exception(exc) + return ChatAdapterResult( + ok=False, + error_type=error_type, + http_status=status, + edge_fingerprint=edge, + ) + + +class ChatGateway: + def __init__(self, adapter, circuit, settings, route_adapters=None) -> None: + self._adapter = adapter + self._circuit = circuit + self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHAT.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return lookup_adapter(self._route_adapters, route, self._adapter) + + def _models(self) -> tuple[str, ...]: + if not self._settings.model.strip(): + raise RuntimeError("chat gateway requires AI_MODEL") + return (self._settings.model, *_parse_fallbacks(self._settings.chat_fallbacks)) + + def invoke(self, messages: Any, **kwargs: Any) -> Any: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = _hash_messages(messages) + models = self._models() + fallback_used = False + fallback_reason: str | None = None + route_reason_override: str | None = None + last_http_status: int | None = None + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + raise RuntimeError( + f"chat gateway failed request_id={request_id} http_status={http_status}" + ) + + if self._circuit.is_open("aggregator"): + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=models[0], + family=Family.CHAT_COMPLETIONS.value, + route=self._routes[0], + attempt_index=0, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + fail(None) + + for route_index, route in enumerate(self._routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(self._routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(self._routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) + + adapter = self._adapter_for(route) + switch_to_next_route = False + for model_index, model in enumerate(models): + if model_index == 0: + route_reason = route_reason_override or "primary" + elif fallback_reason == "429": + route_reason = "fallback_after_429" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + result = adapter.invoke(messages, model=model, **kwargs) + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + if result.ok: + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=model, + family=Family.CHAT_COMPLETIONS.value, + route=route, + attempt_index=model_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="fallback_success" if fallback_used else "success", + http_status=result.http_status, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=True, + detail=AttemptDetail( + input_hash=input_hash, + output_bytes=len(str(result.value).encode()), + retry_after_ms=retry_after_ms, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + return result.value + + error_type = result.error_type or ModelErrorType.UNKNOWN + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=False, + ) + has_next_route = route_index + 1 < len(self._routes) + circuit_scope = None + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=model, + family=Family.CHAT_COMPLETIONS.value, + route=route, + attempt_index=model_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type.value, + http_status=result.http_status, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=False, + detail=AttemptDetail( + input_hash=input_hash, + retry_after_ms=retry_after_ms, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + if step is NextStep.OPEN_AGGREGATOR and has_next_route: + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.FALLBACK_KEY: + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + break + fail(last_http_status) + if switch_to_next_route: + break + if switch_to_next_route: + continue + route_reason_override = None + + fail(last_http_status) + + def _emit(self, trace: AttemptTrace) -> None: + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) + + +def build_chat_gateway(config=None, *, adapter=None, circuit=None, **client_kwargs: Any) -> ChatGateway: + cfg: AIProviderSettings = config or default_settings + route_adapters = None + if adapter is None: + routes = routes_from_settings(cfg, route_group=Scene.CHAT.value) + route_adapters = { + route.route_id: LangChainChatAdapter(config_for_route(cfg, route), **client_kwargs) + for route in routes + } + adapter = route_adapters[routes[0].route_id] + return ChatGateway( + adapter=adapter, + circuit=circuit if circuit is not None else _CIRCUIT, + settings=cfg, + route_adapters=route_adapters, + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py index 7e9ae23c..3ab36e85 100644 --- a/backend/packages/framework/src/windup_framework/gateway/image.py +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -9,14 +9,16 @@ from windup_framework.gateway.circuit import CircuitBreaker from windup_framework.gateway.context import current_call_context from windup_framework.gateway.policy import decide -from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.registry import ModelRegistry, RegistryError from windup_framework.gateway.routes import ( GatewayRoute, config_for_route, - route_layer_for, + key_circuit_id, + lookup_adapter, routes_from_settings, ) from windup_framework.gateway.trace import ( + AttemptDetail, AttemptTrace, emit, estimate_cost, @@ -44,7 +46,7 @@ def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> self._route_adapters = dict(route_adapters or {}) def _adapter_for(self, route: GatewayRoute): - return self._route_adapters.get(route.base_url_id, self._adapter) + return lookup_adapter(self._route_adapters, route, self._adapter) def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: ctx = current_call_context() @@ -77,18 +79,19 @@ def fail(http_status: int | None) -> None: model = models[0] if models else "" route = routes[0] self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=start_i, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="aggregator", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=False, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) ) fail(None) @@ -99,6 +102,12 @@ def fail(http_status: int | None) -> None: route_reason_override = "base_url_unreached" continue fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) adapter = self._adapter_for(route) switch_to_next_route = False @@ -108,18 +117,20 @@ def fail(http_status: int | None) -> None: fallback_used = True fallback_reason = "skip" self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=attempt_index, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="model", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="model", + fallback_used=fallback_used, + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) ) continue @@ -160,33 +171,36 @@ def fail(http_status: int | None) -> None: ) if result.ok: self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - circuit_scope=None, - outcome="fallback_success" if fallback_used else "success", - input_hash=input_hash, - output_hash=hash_bytes(result.body), - total_latency_ms=total_ms(), - fallback_used=fallback_used, - http_status=result.http_status, - maybe_billed=True, - cost=cost, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - output_bytes=len(result.body), - expected_bytes=result.expected_bytes, - provider_usage=result.provider_usage, - edge_fingerprint=result.edge_fingerprint or None, - job_id=result.job_id, - job_status=result.job_status, - retry_after_ms=retry_after_ms, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="fallback_success" if fallback_used else "success", + http_status=result.http_status, + job_id=result.job_id, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=True, + cost=cost, + detail=AttemptDetail( + input_hash=input_hash, + output_hash=hash_bytes(result.body), + output_bytes=len(result.body), + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) ) return result.body @@ -205,45 +219,58 @@ def fail(http_status: int | None) -> None: else: self._circuit.open("aggregator") circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" elif step is NextStep.FALLBACK: self._circuit.open("model:" + model) circuit_scope = "model" self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - circuit_scope=circuit_scope, - outcome="failed", - input_hash=input_hash, - output_hash=None, - total_latency_ms=total_ms(), - fallback_used=fallback_used, - http_status=result.http_status, - error_type=error_type.value, - maybe_billed=result.maybe_billed, - cost=cost, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - output_bytes=result.output_bytes or None, - expected_bytes=result.expected_bytes, - provider_usage=result.provider_usage, - edge_fingerprint=result.edge_fingerprint or None, - job_id=result.job_id, - job_status=result.job_status, - retry_after_ms=retry_after_ms, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type.value, + http_status=result.http_status, + job_id=result.job_id, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=result.maybe_billed, + cost=cost, + detail=AttemptDetail( + input_hash=input_hash, + output_bytes=result.output_bytes or None, + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) ) if step is NextStep.OPEN_AGGREGATOR and has_next_route: fallback_used = True route_reason_override = "base_url_unreached" switch_to_next_route = True break + if step is NextStep.FALLBACK_KEY: + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: wait = ( @@ -271,89 +298,19 @@ def fail(http_status: int | None) -> None: fail(last_http_status) - def _emit( - self, - *, - request_id: str, - ctx, - model: str, - route: GatewayRoute, - attempt_index: int, - retry_count: int, - route_reason: str, - circuit_scope: str | None, - outcome: str, - input_hash: str, - total_latency_ms: int, - fallback_used: bool, - output_hash: str | None = None, - http_status: int | None = None, - error_type: str | None = None, - maybe_billed: bool | None = None, - cost: float | None = None, - started_at: str | None = None, - ended_at: str | None = None, - attempt_latency_ms: int | None = None, - resend_spent: int | None = 0, - output_bytes: int | None = None, - expected_bytes: int | None = None, - provider_usage: object | None = None, - edge_fingerprint: str | None = None, - job_id: str | None = None, - job_status: str | None = None, - retry_after_ms: int | None = None, - ) -> None: - family = None - if model: - family = self._registry.family_of(model).value - emit( - AttemptTrace( - request_id=request_id, - attempt_id=str(uuid.uuid4()), - task_id=ctx.task_id, - user_id=ctx.user_id, - scene=Scene.CHARACTER_IMAGE, - model=model, - family=family, - route_id=route.route_id, - route_group=route.route_group, - candidate_index=route.candidate_index, - provider_name=route.provider_name, - base_url_id=route.base_url_id, - base_url_host=route.host, - api_key_id=route.api_key_id, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - route_layer=route_layer_for(route_reason), - circuit_scope=circuit_scope, - error_type=error_type, - http_status=http_status, - edge_fingerprint=edge_fingerprint, - job_id=job_id, - fallback_used=fallback_used, - outcome=outcome, - job_status=job_status, - started_at=started_at or _utc_now(), - ended_at=ended_at or _utc_now(), - attempt_latency_ms=attempt_latency_ms, - total_latency_ms=total_latency_ms, - submit_ms=None, - poll_ms=None, - download_ms=None, - poll_count=None, - retry_after_ms=retry_after_ms, - resend_spent=resend_spent, - output_bytes=output_bytes, - expected_bytes=expected_bytes, - input_hash=input_hash, - output_hash=output_hash, - maybe_billed=maybe_billed, - cost=cost, - price_version=self._settings.price_version, - provider_usage=provider_usage, - ) - ) + def _emit(self, trace: AttemptTrace) -> None: + if not trace.family and trace.model: + try: + trace.family = self._registry.family_of(trace.model).value + except RegistryError: + pass + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) def build_image_gateway(config=None, *, adapter=None, circuit=None) -> ImageGateway: @@ -364,10 +321,10 @@ def build_image_gateway(config=None, *, adapter=None, circuit=None) -> ImageGate routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_IMAGE.value) route_adapters = { - route.base_url_id: SufyImageProvider(config=config_for_route(cfg, route)) + route.route_id: SufyImageProvider(config=config_for_route(cfg, route)) for route in routes } - adapter = route_adapters[routes[0].base_url_id] + adapter = route_adapters[routes[0].route_id] return ImageGateway( ModelRegistry.from_settings(cfg), adapter, diff --git a/backend/packages/framework/src/windup_framework/gateway/ledger.py b/backend/packages/framework/src/windup_framework/gateway/ledger.py index 4bfb700f..e0f0f468 100644 --- a/backend/packages/framework/src/windup_framework/gateway/ledger.py +++ b/backend/packages/framework/src/windup_framework/gateway/ledger.py @@ -8,7 +8,8 @@ from windup_framework.db import SessionLocal from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail -from windup_framework.gateway.trace import AttemptTrace +from windup_framework.gateway.routes import route_layer_for +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace logger = logging.getLogger("windup.gateway.ledger") @@ -43,6 +44,14 @@ def _ledger_outcome(value: str | None) -> str: return "failed" +def _phase_for(trace: AttemptTrace) -> str: + if trace.scene.value == "chat": + return "chat_sync" + if trace.scene.value == "character_image": + return "image_sync" + return "submit" + + def _json_or_none(value: Any) -> Any: if value is None: return None @@ -55,6 +64,7 @@ def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> Non """Persist one gateway attempt without letting ledger failures affect generation.""" attempt_uuid = _uuid(trace.attempt_id) + route = trace.route try: with session_factory() as session: session.add( @@ -67,19 +77,19 @@ def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> Non scene=trace.scene.value, attempt_index=trace.attempt_index or 0, retry_count=trace.retry_count, - route_id=trace.route_id or "default.primary", - route_group=trace.route_group or trace.scene.value, - candidate_index=trace.candidate_index or 0, - provider_name=trace.provider_name or "openai-compatible", - base_url_id=trace.base_url_id or "primary", - base_url_host=trace.base_url_host or "", - api_key_id=trace.api_key_id, + route_id=route.route_id, + route_group=route.route_group, + candidate_index=route.candidate_index, + provider_name=route.provider_name, + base_url_id=route.base_url_id, + base_url_host=route.host or "", + api_key_id=route.api_key_id, model=trace.model, family=trace.family or "", route_reason=trace.route_reason or "primary", - route_layer=trace.route_layer or "none", + route_layer=route_layer_for(trace.route_reason), circuit_scope=trace.circuit_scope, - phase="image_sync" if trace.scene.value == "character_image" else "submit", + phase=_phase_for(trace), outcome=_ledger_outcome(trace.outcome), job_id=trace.job_id, error_type=trace.error_type, @@ -98,6 +108,7 @@ def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> Non logger.exception("Gateway hot ledger write failed request_id=%s", trace.request_id) return + detail = trace.detail or AttemptDetail() try: with session_factory() as session: session.add( @@ -105,20 +116,20 @@ def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> Non attempt_id=attempt_uuid, request_id=trace.request_id, task_id=_int_or_none(trace.task_id), - job_status=trace.job_status, - edge_fingerprint=trace.edge_fingerprint, + job_status=detail.job_status, + edge_fingerprint=detail.edge_fingerprint, error_message=None, provider_request_id=None, - provider_usage=_json_or_none(trace.provider_usage), - input_hash=trace.input_hash, - output_hash=trace.output_hash, - output_bytes=trace.output_bytes, - expected_bytes=trace.expected_bytes, - retry_after_ms=trace.retry_after_ms, - submit_ms=trace.submit_ms, - poll_ms=trace.poll_ms, - download_ms=trace.download_ms, - poll_count=trace.poll_count, + provider_usage=_json_or_none(detail.provider_usage), + input_hash=detail.input_hash, + output_hash=detail.output_hash, + output_bytes=detail.output_bytes, + expected_bytes=detail.expected_bytes, + retry_after_ms=detail.retry_after_ms, + submit_ms=detail.submit_ms, + poll_ms=detail.poll_ms, + download_ms=detail.download_ms, + poll_count=detail.poll_count, extra=None, ) ) diff --git a/backend/packages/framework/src/windup_framework/gateway/models.py b/backend/packages/framework/src/windup_framework/gateway/models.py index 95cf95d7..1b22588a 100644 --- a/backend/packages/framework/src/windup_framework/gateway/models.py +++ b/backend/packages/framework/src/windup_framework/gateway/models.py @@ -43,11 +43,11 @@ class AIGatewayAttempt(Base): __tablename__ = "windup_ai_gateway_attempt" __table_args__ = ( CheckConstraint( - "scene IN ('character_image', 'character_action')", + "scene IN ('chat', 'character_image', 'character_action')", name="ck_gateway_attempt_scene", ), CheckConstraint( - "phase IN ('image_sync', 'submit', 'follow', 'download')", + "phase IN ('chat_sync', 'image_sync', 'submit', 'follow', 'download')", name="ck_gateway_attempt_phase", ), CheckConstraint( diff --git a/backend/packages/framework/src/windup_framework/gateway/policy.py b/backend/packages/framework/src/windup_framework/gateway/policy.py index 9a98c2d7..181b4ea7 100644 --- a/backend/packages/framework/src/windup_framework/gateway/policy.py +++ b/backend/packages/framework/src/windup_framework/gateway/policy.py @@ -25,7 +25,7 @@ def decide( if error_type is ModelErrorType.RATE_LIMIT and retry_count < 2: return NextStep.RETRY_SAME if error_type is ModelErrorType.RATE_LIMIT: - return NextStep.FALLBACK + return NextStep.FALLBACK_KEY if error_type is ModelErrorType.INVALID_RESPONSE and retry_count < 2: return NextStep.RETRY_SAME if error_type is ModelErrorType.INVALID_RESPONSE: diff --git a/backend/packages/framework/src/windup_framework/gateway/routes.py b/backend/packages/framework/src/windup_framework/gateway/routes.py index 7438ada2..89db9aae 100644 --- a/backend/packages/framework/src/windup_framework/gateway/routes.py +++ b/backend/packages/framework/src/windup_framework/gateway/routes.py @@ -22,32 +22,68 @@ def host(self) -> str | None: return urlparse(self.base_url).hostname +def _parse_csv(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +def _unique_keys(first: str, extra: str) -> tuple[str, ...]: + keys: list[str] = [] + for key in (first.strip(), *_parse_csv(extra)): + if key and key not in keys: + keys.append(key) + return tuple(keys) or ("",) + + +def _expand_url( + *, + route_group: str, + provider_name: str, + base_url_id: str, + base_url: str, + first_key: str, + extra_keys: str, + start_index: int, +) -> list[GatewayRoute]: + routes: list[GatewayRoute] = [] + for i, api_key in enumerate(_unique_keys(first_key, extra_keys)): + api_key_id = f"{base_url_id}.key{i}" + routes.append( + GatewayRoute( + route_id=api_key_id, + route_group=route_group, + candidate_index=start_index + i, + provider_name=provider_name, + base_url_id=base_url_id, + base_url=base_url, + api_key_id=api_key_id, + api_key=api_key, + ) + ) + return routes + + def routes_from_settings(cfg: AIProviderSettings, *, route_group: str) -> tuple[GatewayRoute, ...]: primary_name = cfg.route_primary_name.strip() or "primary" - routes = [ - GatewayRoute( - route_id=f"{primary_name}.primary", - route_group=route_group, - candidate_index=0, - provider_name=cfg.provider, - base_url_id=primary_name, - base_url=cfg.effective_route_primary_base_url, - api_key_id=primary_name, - api_key=cfg.effective_route_primary_api_key, - ) - ] + routes = _expand_url( + route_group=route_group, + provider_name=cfg.provider, + base_url_id=primary_name, + base_url=cfg.effective_route_primary_base_url, + first_key=cfg.effective_route_primary_api_key, + extra_keys=cfg.route_primary_api_keys, + start_index=0, + ) if cfg.route_fallback_enabled: fallback_name = cfg.route_fallback_name.strip() - routes.append( - GatewayRoute( - route_id=f"{fallback_name}.fallback", + routes.extend( + _expand_url( route_group=route_group, - candidate_index=1, provider_name=cfg.provider, base_url_id=fallback_name, base_url=cfg.route_fallback_base_url.rstrip("/"), - api_key_id=fallback_name, - api_key=cfg.route_fallback_api_key, + first_key=cfg.route_fallback_api_key, + extra_keys=cfg.route_fallback_api_keys, + start_index=len(routes), ) ) return tuple(routes) @@ -57,9 +93,24 @@ def config_for_route(cfg: AIProviderSettings, route: GatewayRoute) -> AIProvider return cfg.model_copy(update={"base_url": route.base_url, "api_key": route.api_key}) +def lookup_adapter(route_adapters: dict, route: GatewayRoute, default): + return ( + route_adapters.get(route.route_id) + or route_adapters.get(route.api_key_id) + or route_adapters.get(route.base_url_id) + or default + ) + + +def key_circuit_id(route: GatewayRoute) -> str: + return f"key:{route.base_url_id}:{route.api_key_id}" + + def route_layer_for(reason: str) -> str: if reason == "base_url_unreached": return "base_url" + if reason == "key_rate_limit": + return "key" if reason in { "fallback_after_429", "fallback_after_upstream_fail", diff --git a/backend/packages/framework/src/windup_framework/gateway/trace.py b/backend/packages/framework/src/windup_framework/gateway/trace.py index 4a37050c..b8149d33 100644 --- a/backend/packages/framework/src/windup_framework/gateway/trace.py +++ b/backend/packages/framework/src/windup_framework/gateway/trace.py @@ -3,69 +3,83 @@ import hashlib import json import logging +import uuid from dataclasses import dataclass, fields from enum import Enum from windup_framework.config.provider import settings as provider_settings +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.routes import GatewayRoute, route_layer_for from windup_framework.gateway.types import Scene logger = logging.getLogger("windup.gateway") +@dataclass +class AttemptDetail: + input_hash: str | None = None + output_hash: str | None = None + output_bytes: int | None = None + expected_bytes: int | None = None + retry_after_ms: int | None = None + submit_ms: int | None = None + poll_ms: int | None = None + download_ms: int | None = None + poll_count: int | None = None + resend_spent: int | None = None + job_status: str | None = None + edge_fingerprint: str | None = None + provider_usage: object | None = None + + @dataclass class AttemptTrace: request_id: str scene: Scene model: str + route: GatewayRoute + attempt_index: int + retry_count: int + route_reason: str + outcome: str attempt_id: str | None = None task_id: str | None = None user_id: str | None = None family: str | None = None - route_id: str | None = None - route_group: str | None = None - candidate_index: int | None = None - provider_name: str | None = None - base_url_id: str | None = None - base_url_host: str | None = None - api_key_id: str | None = None - attempt_index: int | None = None - retry_count: int = 0 - route_reason: str | None = None - route_layer: str | None = None circuit_scope: str | None = None error_type: str | None = None http_status: int | None = None - edge_fingerprint: str | None = None job_id: str | None = None fallback_used: bool = False - outcome: str | None = None - job_status: str | None = None started_at: str | None = None ended_at: str | None = None attempt_latency_ms: int | None = None total_latency_ms: int | None = None - submit_ms: int | None = None - poll_ms: int | None = None - download_ms: int | None = None - poll_count: int | None = None - retry_after_ms: int | None = None - resend_spent: int | None = None - output_bytes: int | None = None - expected_bytes: int | None = None - input_hash: str | None = None - output_hash: str | None = None maybe_billed: bool | None = None cost: float | None = None price_version: str | None = None - provider_usage: object | None = None + detail: AttemptDetail | None = None def as_dict(self) -> dict[str, object]: out: dict[str, object] = {} for f in fields(self): + if f.name in {"route", "detail"}: + continue value = getattr(self, f.name) if isinstance(value, Enum): value = value.value out[f.name] = value + out["route_id"] = self.route.route_id + out["route_group"] = self.route.route_group + out["candidate_index"] = self.route.candidate_index + out["provider_name"] = self.route.provider_name + out["base_url_id"] = self.route.base_url_id + out["base_url_host"] = self.route.host + out["api_key_id"] = self.route.api_key_id + out["route_layer"] = route_layer_for(self.route_reason) + detail = self.detail or AttemptDetail() + for f in fields(detail): + out[f.name] = getattr(detail, f.name) return out @@ -98,6 +112,15 @@ def hash_bytes(data: bytes) -> str: def emit(trace: AttemptTrace) -> None: + ctx = current_call_context() + if not trace.attempt_id: + trace.attempt_id = str(uuid.uuid4()) + if trace.task_id is None: + trace.task_id = ctx.task_id + if trace.user_id is None: + trace.user_id = ctx.user_id + if trace.price_version is None: + trace.price_version = provider_settings.price_version logger.info("%s", json.dumps(trace.as_dict(), ensure_ascii=False, default=str)) if provider_settings.gateway_ledger_enabled: from windup_framework.gateway.ledger import persist_attempt diff --git a/backend/packages/framework/src/windup_framework/gateway/types.py b/backend/packages/framework/src/windup_framework/gateway/types.py index d056238c..2566b51b 100644 --- a/backend/packages/framework/src/windup_framework/gateway/types.py +++ b/backend/packages/framework/src/windup_framework/gateway/types.py @@ -7,11 +7,13 @@ class Scene(str, Enum): + CHAT = "chat" CHARACTER_IMAGE = "character_image" CHARACTER_ACTION = "character_action" class Family(str, Enum): + CHAT_COMPLETIONS = "chat.completions" IMAGE_CHAT_DATA_URI = "image.chat_data_uri" VIDEO_INPUT_REFERENCE = "video.input_reference" VIDEO_IMAGE_LIST = "video.image_list" @@ -20,6 +22,7 @@ class Family(str, Enum): class NextStep(str, Enum): RETRY_SAME = "retry_same" FALLBACK = "fallback" + FALLBACK_KEY = "fallback_key" FAIL = "fail" OPEN_AGGREGATOR = "open_aggregator" diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py index ff82c683..643b26e1 100644 --- a/backend/packages/framework/src/windup_framework/gateway/video.py +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -10,14 +10,16 @@ from windup_framework.gateway.context import current_call_context from windup_framework.gateway.image import _CIRCUIT from windup_framework.gateway.policy import decide -from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.registry import ModelRegistry, RegistryError from windup_framework.gateway.routes import ( GatewayRoute, config_for_route, - route_layer_for, + key_circuit_id, + lookup_adapter, routes_from_settings, ) from windup_framework.gateway.trace import ( + AttemptDetail, AttemptTrace, emit, estimate_cost, @@ -44,7 +46,7 @@ def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> self._route_adapters = dict(route_adapters or {}) def _adapter_for(self, route: GatewayRoute): - return self._route_adapters.get(route.base_url_id, self._adapter) + return lookup_adapter(self._route_adapters, route, self._adapter) def i2v( self, @@ -86,18 +88,19 @@ def fail(http_status: int | None) -> None: model = models[0] if models else "" route = routes[0] self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=start_i, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="aggregator", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=False, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_ACTION, + model=model, + route=route, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) ) fail(None) @@ -108,6 +111,12 @@ def fail(http_status: int | None) -> None: route_reason_override = "base_url_unreached" continue fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) adapter = self._adapter_for(route) switch_to_next_route = False @@ -117,18 +126,20 @@ def fail(http_status: int | None) -> None: fallback_used = True fallback_reason = "skip" self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=attempt_index, - retry_count=0, - route_reason="skip_circuit_open", - circuit_scope="model", - outcome="failed", - input_hash=input_hash, - total_latency_ms=total_ms(), - fallback_used=fallback_used, + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_ACTION, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="model", + fallback_used=fallback_used, + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) ) continue @@ -177,7 +188,6 @@ def fail(http_status: int | None) -> None: if result.ok: self._emit_result( request_id=request_id, - ctx=ctx, model=model, route=route, attempt_index=attempt_index, @@ -214,13 +224,15 @@ def fail(http_status: int | None) -> None: else: self._circuit.open("aggregator") circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" elif step is NextStep.FALLBACK: self._circuit.open("model:" + model) circuit_scope = "model" self._emit_result( request_id=request_id, - ctx=ctx, model=model, route=route, attempt_index=attempt_index, @@ -249,6 +261,15 @@ def fail(http_status: int | None) -> None: route_reason_override = "base_url_unreached" switch_to_next_route = True break + if step is NextStep.FALLBACK_KEY: + if bound_job_id is not None: + fail(last_http_status) + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) if step is NextStep.RETRY_SAME: if error_type is ModelErrorType.RATE_LIMIT: wait = ( @@ -291,7 +312,6 @@ def _emit_result( self, *, request_id: str, - ctx, model: str, route: GatewayRoute, attempt_index: int, @@ -325,128 +345,58 @@ def _emit_result( else None ) self._emit( - request_id=request_id, - ctx=ctx, - model=model, - route=route, - attempt_index=attempt_index, - retry_count=retry_count, - route_reason=route_reason, - circuit_scope=circuit_scope, - outcome=outcome, - input_hash=input_hash, - output_hash=hash_bytes(result.body) if result.ok else None, - total_latency_ms=total_latency_ms, - fallback_used=fallback_used, - http_status=result.http_status, - error_type=error_type.value if error_type is not None else None, - maybe_billed=True if result.ok else result.maybe_billed, - cost=cost, - started_at=started_at, - ended_at=ended_at, - attempt_latency_ms=attempt_latency_ms, - resend_spent=resend_spent, - output_bytes=len(result.body) if result.ok else (result.output_bytes or None), - expected_bytes=result.expected_bytes, - provider_usage=result.provider_usage, - edge_fingerprint=result.edge_fingerprint or None, - job_id=result.job_id, - job_status=result.job_status, - retry_after_ms=retry_after_ms, - submit_ms=submit_ms, - poll_ms=result.poll_ms, - download_ms=result.download_ms, - poll_count=result.poll_count, - ) - - def _emit( - self, - *, - request_id: str, - ctx, - model: str, - route: GatewayRoute, - attempt_index: int, - retry_count: int, - route_reason: str, - circuit_scope: str | None, - outcome: str, - input_hash: str, - total_latency_ms: int, - fallback_used: bool, - output_hash: str | None = None, - http_status: int | None = None, - error_type: str | None = None, - maybe_billed: bool | None = None, - cost: float | None = None, - started_at: str | None = None, - ended_at: str | None = None, - attempt_latency_ms: int | None = None, - resend_spent: int | None = 0, - output_bytes: int | None = None, - expected_bytes: int | None = None, - provider_usage: object | None = None, - edge_fingerprint: str | None = None, - job_id: str | None = None, - job_status: str | None = None, - retry_after_ms: int | None = None, - submit_ms: int | None = None, - poll_ms: int | None = None, - download_ms: int | None = None, - poll_count: int | None = None, - ) -> None: - family = None - if model: - family = self._registry.family_of(model).value - emit( AttemptTrace( request_id=request_id, - attempt_id=str(uuid.uuid4()), - task_id=ctx.task_id, - user_id=ctx.user_id, scene=Scene.CHARACTER_ACTION, model=model, - family=family, - route_id=route.route_id, - route_group=route.route_group, - candidate_index=route.candidate_index, - provider_name=route.provider_name, - base_url_id=route.base_url_id, - base_url_host=route.host, - api_key_id=route.api_key_id, + route=route, attempt_index=attempt_index, retry_count=retry_count, route_reason=route_reason, - route_layer=route_layer_for(route_reason), + outcome=outcome, circuit_scope=circuit_scope, - error_type=error_type, - http_status=http_status, - edge_fingerprint=edge_fingerprint, - job_id=job_id, + error_type=error_type.value if error_type is not None else None, + http_status=result.http_status, + job_id=result.job_id, fallback_used=fallback_used, - outcome=outcome, - job_status=job_status, - started_at=started_at or _utc_now(), - ended_at=ended_at or _utc_now(), + started_at=started_at, + ended_at=ended_at, attempt_latency_ms=attempt_latency_ms, total_latency_ms=total_latency_ms, - submit_ms=submit_ms, - poll_ms=poll_ms, - download_ms=download_ms, - poll_count=poll_count, - retry_after_ms=retry_after_ms, - resend_spent=resend_spent, - output_bytes=output_bytes, - expected_bytes=expected_bytes, - input_hash=input_hash, - output_hash=output_hash, - maybe_billed=maybe_billed, + maybe_billed=True if result.ok else result.maybe_billed, cost=cost, - price_version=self._settings.price_version, - provider_usage=provider_usage, + detail=AttemptDetail( + input_hash=input_hash, + output_hash=hash_bytes(result.body) if result.ok else None, + output_bytes=len(result.body) if result.ok else (result.output_bytes or None), + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + submit_ms=submit_ms, + poll_ms=result.poll_ms, + download_ms=result.download_ms, + poll_count=result.poll_count, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), ) ) + def _emit(self, trace: AttemptTrace) -> None: + if not trace.family and trace.model: + try: + trace.family = self._registry.family_of(trace.model).value + except RegistryError: + pass + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) + def build_video_gateway(config=None, *, adapter=None, circuit=None) -> VideoGateway: cfg: AIProviderSettings = config or default_settings @@ -456,10 +406,10 @@ def build_video_gateway(config=None, *, adapter=None, circuit=None) -> VideoGate routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_ACTION.value) route_adapters = { - route.base_url_id: SufyVideoProvider(config=config_for_route(cfg, route)) + route.route_id: SufyVideoProvider(config=config_for_route(cfg, route)) for route in routes } - adapter = route_adapters[routes[0].base_url_id] + adapter = route_adapters[routes[0].route_id] return VideoGateway( ModelRegistry.from_settings(cfg), adapter, diff --git a/backend/packages/framework/src/windup_framework/providers/chat.py b/backend/packages/framework/src/windup_framework/providers/chat.py index acf98709..606d816f 100644 --- a/backend/packages/framework/src/windup_framework/providers/chat.py +++ b/backend/packages/framework/src/windup_framework/providers/chat.py @@ -2,25 +2,17 @@ from typing import Any -from langchain_openai import ChatOpenAI - from windup_framework.config.provider import AIProviderSettings, settings +from windup_framework.gateway.chat import ChatGateway, build_chat_gateway def create_chat_model( config: AIProviderSettings = settings, **kwargs: Any, -) -> ChatOpenAI: - """创建 LangChain 官方 ``ChatOpenAI`` 实例。 +) -> ChatGateway: + """创建带 Gateway 策略的 Chat 模型。 - 这里仅统一 Windup 配置到 LangChain 官方客户端的映射，不重新实现 - ``BaseChatModel``、消息转换、工具调用或结构化输出。 + 协议适配仍由 LangChain 官方 ``ChatOpenAI`` 完成；Gateway 只负责 + route / retry / circuit / trace。 """ - return ChatOpenAI( - model=config.model, - api_key=config.api_key or None, - base_url=config.normalized_base_url, - timeout=config.timeout, - max_retries=config.max_retries, - **kwargs, - ) + return build_chat_gateway(config=config, **kwargs) diff --git a/backend/tests/test_gateway_chat.py b/backend/tests/test_gateway_chat.py new file mode 100644 index 00000000..94340f6e --- /dev/null +++ b/backend/tests/test_gateway_chat.py @@ -0,0 +1,95 @@ +import json +import logging + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.chat import ChatAdapterResult, ChatGateway +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.providers.chat import create_chat_model + +UNREACHED = ChatAdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +OK = ChatAdapterResult(ok=True, value="pong") + + +class FakeChatAdapter: + def __init__(self, by_model: dict[str, list[ChatAdapterResult]]): + self.by_model = {k: list(v) for k, v in by_model.items()} + self.calls: list[str] = [] + + def invoke(self, messages, *, model: str, **kwargs): + self.calls.append(model) + q = self.by_model[model] + return q.pop(0) if q else ChatAdapterResult(ok=False, error_type=ModelErrorType.UNKNOWN) + + +def test_chat_gateway_switches_base_url_route_after_unreached(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeChatAdapter({"gpt-4o-mini": [UNREACHED, UNREACHED]}) + backup = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = AIProviderSettings( + model="gpt-4o-mini", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = ChatGateway( + adapter=primary, + circuit=CircuitBreaker(), + settings=cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert primary.calls == ["gpt-4o-mini", "gpt-4o-mini"] + assert backup.calls == ["gpt-4o-mini"] + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["scene"] == "chat" + assert line["family"] == "chat.completions" + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + +def test_chat_gateway_switches_key_after_429(monkeypatch, caplog): + monkeypatch.setattr("windup_framework.gateway.chat.time.sleep", lambda _: None) + caplog.set_level(logging.INFO, logger="windup.gateway") + rate = ChatAdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeChatAdapter({"gpt-4o-mini": [rate, rate, rate]}) + key_b = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = AIProviderSettings( + model="gpt-4o-mini", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = ChatGateway( + adapter=key_a, + circuit=CircuitBreaker(), + settings=cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert key_a.calls == ["gpt-4o-mini"] * 3 + assert key_b.calls == ["gpt-4o-mini"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + line = success[-1] + assert line["route_reason"] == "key_rate_limit" + assert line["route_layer"] == "key" + + +def test_create_chat_model_returns_gateway_without_hand_rolling_protocol(): + cfg = AIProviderSettings(model="gpt-4o-mini") + chat = create_chat_model(config=cfg) + + assert hasattr(chat, "invoke") + assert chat.__class__.__name__ == "ChatGateway" diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py index 9a84811c..e4448361 100644 --- a/backend/tests/test_gateway_image.py +++ b/backend/tests/test_gateway_image.py @@ -99,7 +99,7 @@ def test_aggregator_circuit_skips_fallback_model(): assert br.is_open("aggregator") -def test_429_falls_back_to_next_model(monkeypatch): +def test_429_does_not_switch_model_when_only_one_key(monkeypatch): monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) ad = FakeImageAdapter({ @@ -107,9 +107,83 @@ def test_429_falls_back_to_next_model(monkeypatch): "gemini-2.5-flash-image-alt": [PNG], }) gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + with pytest.raises(RuntimeError, match="429"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image"] * 3 + assert "gemini-2.5-flash-image-alt" not in ad.calls + + +def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + caplog.set_level(logging.INFO, logger="windup.gateway") + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeImageAdapter({ + "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image-alt": [PNG], + }) + key_b = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + key_a, + CircuitBreaker(), + cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + assert gw.gen_image("p", []).startswith(b"\x89PNG") - assert ad.calls[-1] == "gemini-2.5-flash-image-alt" - assert ad.calls.count("gemini-2.5-flash-image") == 3 + assert key_a.calls == ["gemini-2.5-flash-image"] * 3 + assert key_b.calls == ["gemini-2.5-flash-image"] + assert "gemini-2.5-flash-image-alt" not in key_a.calls + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "key_rate_limit" + assert line["route_layer"] == "key" + assert line["base_url_id"] == "primary" + assert line["api_key_id"].endswith("key1") + + +def test_522_skips_remaining_keys_on_same_url(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + key_a = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + key_b = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + backup = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="key-c", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + key_a, + CircuitBreaker(), + cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b, "backup.key0": backup}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert key_a.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + assert key_b.calls == [] + assert backup.calls == ["gemini-2.5-flash-image"] def test_520_does_not_retry(): diff --git a/backend/tests/test_gateway_ledger_persistence.py b/backend/tests/test_gateway_ledger_persistence.py index 3b995e09..2f3eab29 100644 --- a/backend/tests/test_gateway_ledger_persistence.py +++ b/backend/tests/test_gateway_ledger_persistence.py @@ -9,7 +9,8 @@ from windup_framework.db import Base from windup_framework.gateway.ledger import persist_attempt from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail -from windup_framework.gateway.trace import AttemptTrace +from windup_framework.gateway.routes import GatewayRoute +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace from windup_framework.gateway.types import Scene @@ -31,27 +32,30 @@ def test_persist_attempt_splits_hot_and_detail_fields(): scene=Scene.CHARACTER_IMAGE, model="gemini-2.5-flash-image", family="image.chat_data_uri", - route_id="backup.fallback", - route_group="character_image", - candidate_index=1, - provider_name="openai-compatible", - base_url_id="backup", - base_url_host="backup.example.com", - api_key_id="backup", + route=GatewayRoute( + route_id="backup.fallback", + route_group="character_image", + candidate_index=1, + provider_name="openai-compatible", + base_url_id="backup", + base_url="https://backup.example.com/v1", + api_key_id="backup", + api_key="k", + ), attempt_index=2, retry_count=1, route_reason="base_url_unreached", - route_layer="base_url", - circuit_scope=None, outcome="fallback_success", - edge_fingerprint="cf-ray=abc", maybe_billed=True, cost=0.25, price_version="2026-08-16", - provider_usage={"total_tokens": 12}, started_at=datetime.now(timezone.utc).isoformat(), ended_at=datetime.now(timezone.utc).isoformat(), attempt_latency_ms=123, + detail=AttemptDetail( + edge_fingerprint="cf-ray=abc", + provider_usage={"total_tokens": 12}, + ), ), session_factory=session_factory, ) diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py index 5d58a165..ddaac166 100644 --- a/backend/tests/test_gateway_policy.py +++ b/backend/tests/test_gateway_policy.py @@ -12,10 +12,10 @@ def test_522_retries_once_then_opens_aggregator(): assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=False) is NextStep.OPEN_AGGREGATOR -def test_429_retries_twice_then_fallback(): +def test_429_retries_twice_then_fallback_key(): assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=1, has_job_id=False) is NextStep.RETRY_SAME - assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK_KEY def test_520_never_retries(): diff --git a/backend/tests/test_gateway_route_config.py b/backend/tests/test_gateway_route_config.py index ad4ffef1..e4117335 100644 --- a/backend/tests/test_gateway_route_config.py +++ b/backend/tests/test_gateway_route_config.py @@ -1,6 +1,7 @@ from __future__ import annotations from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.routes import routes_from_settings def test_gateway_route_env_fields_are_live(): @@ -35,3 +36,26 @@ def test_empty_gateway_route_values_disable_fallback_route(): assert cfg.effective_route_primary_base_url == "https://api.qnaigc.com/v1" assert cfg.effective_route_primary_api_key == "legacy-key" assert cfg.route_fallback_enabled is False + + +def test_routes_expand_extra_keys_on_same_base_url_before_fallback_url(): + cfg = AIProviderSettings( + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="key-c", + ) + routes = routes_from_settings(cfg, route_group="character_image") + + assert [(r.base_url_id, r.api_key, r.base_url) for r in routes] == [ + ("primary", "key-a", "https://api.qnaigc.com/v1"), + ("primary", "key-b", "https://api.qnaigc.com/v1"), + ("backup", "key-c", "https://backup.example.com/v1"), + ] + assert routes[0].api_key_id != routes[1].api_key_id + assert routes[0].candidate_index == 0 + assert routes[1].candidate_index == 1 + assert routes[2].candidate_index == 2 diff --git a/backend/tests/test_gateway_trace.py b/backend/tests/test_gateway_trace.py index 6bf04e44..ed61d484 100644 --- a/backend/tests/test_gateway_trace.py +++ b/backend/tests/test_gateway_trace.py @@ -1,7 +1,10 @@ +import json import logging +from dataclasses import fields from windup_framework.gateway.context import bind_call_context, current_call_context -from windup_framework.gateway.trace import AttemptTrace, emit, estimate_cost +from windup_framework.gateway.routes import GatewayRoute +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace, emit, estimate_cost from windup_framework.gateway.types import Scene REQUIRED = { @@ -17,12 +20,74 @@ "provider_usage", } +COLD_FIELDS = { + "input_hash", "output_hash", "output_bytes", "expected_bytes", + "retry_after_ms", "submit_ms", "poll_ms", "download_ms", "poll_count", + "resend_spent", "job_status", "edge_fingerprint", "provider_usage", +} + + +def _route(**overrides) -> GatewayRoute: + fields_ = dict( + route_id="primary.key0", + route_group="character_image", + candidate_index=0, + provider_name="openai-compatible", + base_url_id="primary", + base_url="https://api.qnaigc.com/v1", + api_key_id="primary.key0", + api_key="k", + ) + fields_.update(overrides) + return GatewayRoute(**fields_) + + +def _trace(**overrides) -> AttemptTrace: + fields_ = dict( + request_id="r1", + scene=Scene.CHARACTER_IMAGE, + model="gemini-2.5-flash-image", + route=_route(), + attempt_index=0, + retry_count=0, + route_reason="primary", + outcome="success", + ) + fields_.update(overrides) + return AttemptTrace(**fields_) + + +def test_cold_fields_live_on_detail_not_trace(): + names = {f.name for f in fields(AttemptTrace)} + assert "route" in names + assert "detail" in names + assert COLD_FIELDS.isdisjoint(names) + assert "route_id" not in names + assert "route_layer" not in names + def test_trace_as_dict_has_required_keys(): - t = AttemptTrace(request_id="r1", scene=Scene.CHARACTER_IMAGE, model="gemini-2.5-flash-image") + t = _trace() keys = set(t.as_dict()) missing = REQUIRED - keys assert not missing, missing + assert "route" not in keys + assert "detail" not in keys + + +def test_as_dict_flattens_route_and_detail(): + t = _trace( + route=_route(base_url_id="backup", route_id="backup.key0"), + route_reason="base_url_unreached", + detail=AttemptDetail(input_hash="abc", submit_ms=12, provider_usage={"n": 1}), + ) + d = t.as_dict() + assert d["base_url_id"] == "backup" + assert d["route_id"] == "backup.key0" + assert d["route_layer"] == "base_url" + assert d["input_hash"] == "abc" + assert d["submit_ms"] == 12 + assert d["provider_usage"] == {"n": 1} def test_cost_null_when_unpriced(): @@ -37,7 +102,7 @@ def test_cost_null_when_unpriced(): def test_cost_never_emits_zero_for_missing_price(): - d = AttemptTrace(request_id="r", scene=Scene.CHARACTER_IMAGE, model="x", cost=None).as_dict() + d = _trace(model="x", cost=None).as_dict() assert d["cost"] is None @@ -52,7 +117,25 @@ def test_context_bind_and_reset(): assert current_call_context().request_id is None +def test_emit_fills_ids_from_context(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + tok = bind_call_context(request_id="abc", task_id="1", user_id="9") + try: + emit(_trace(request_id="r1", family="image.chat_data_uri")) + finally: + tok() + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + assert records + line = records[-1] + assert line["request_id"] == "r1" + assert line["task_id"] == "1" + assert line["user_id"] == "9" + assert line["attempt_id"] + assert line["price_version"] + assert line["family"] == "image.chat_data_uri" + + def test_emit_logs_json_fields(caplog): caplog.set_level(logging.INFO, logger="windup.gateway") - emit(AttemptTrace(request_id="r1", scene=Scene.CHARACTER_IMAGE, model="m")) + emit(_trace(request_id="r1", model="m")) assert "r1" in caplog.text diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py index ebef5fc0..a3885be4 100644 --- a/backend/tests/test_gateway_video.py +++ b/backend/tests/test_gateway_video.py @@ -101,6 +101,45 @@ def test_submit_522_switches_base_url_route_before_model_fallback(caplog): assert line["base_url_id"] == "backup" +def test_submit_429_switches_key_on_same_base_url(monkeypatch): + monkeypatch.setattr("windup_framework.gateway.video.time.sleep", lambda _: None) + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [rate, rate, rate], + "kling-v2-6": [AdapterResult(ok=True, job_id="wrong", maybe_billed=True)], + }, + follows={}, + ) + key_b = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j-key-b", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j-key-b": MP4}, + ) + cfg = AIProviderSettings( + video_model="kling-v2-5-turbo", + video_fallbacks="kling-v2-6", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=key_a, + circuit=CircuitBreaker(cooldown_s=60), + settings=cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + + assert gw.i2v(b"frame", "walk").startswith(b"\x00\x00\x00\x18ftyp") + assert key_a.submit_models == ["kling-v2-5-turbo"] * 3 + assert key_b.submit_models == ["kling-v2-5-turbo"] + assert "kling-v2-6" not in key_a.submit_models + + def test_follow_failed_opens_new_job_on_fallback(): ad = FakeVideoAdapter( submits={ From 6474cab26ab8e9ba1a79d80bc53bbbd034a45e20 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:03:19 +0800 Subject: [PATCH 14/15] =?UTF-8?q?fix(gateway):=20=E5=8E=BB=E6=8E=89?= =?UTF-8?q?=E5=90=88=E5=85=A5=E5=90=8E=E9=87=8D=E5=A4=8D=E7=9A=84=20genera?= =?UTF-8?q?te=20=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 路线选择前误用未赋值的 master，且不应再把型号传给 _get_generator。 --- .../src/windup_app/server/orchestrator/executor.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/backend/packages/app/src/windup_app/server/orchestrator/executor.py b/backend/packages/app/src/windup_app/server/orchestrator/executor.py index 421ad772..1c585e8f 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/executor.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/executor.py @@ -319,9 +319,6 @@ def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) **extra, ) progress: ProgressPort = _LogProgress() - generated = self._get_generator().generate( - card, action, master, progress, canvas=(cons.sprite_w, cons.sprite_h) - ) canvas = (cons.sprite_w, cons.sprite_h) # ── 路线选择:这一步是 server 的事,不是引擎的(#122)──────────────── @@ -332,6 +329,8 @@ def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) # **不静默回退。** 拿到了 model_3d_url 却下载不下来 / 渲不出来,就报错,不改走 # i2v —— 两条路线的画风、成本、多朝向能力都不同,悄悄换一条等于让调用方拿着 # 错误的前提做后续决定,而帧数、时长、成色全都正常,没有任何一道会红。 + # + # 选哪个 kling 不在这里传:run_action_task 已经 bind_call_context(start_from_model)。 model_url = (input.model_3d_url or "").strip() if model_url: rigged = (self._fetch_model3d or self._download_model3d)(model_url) @@ -339,14 +338,12 @@ def _produce_action(self, input: CharacterActionInput, cons: ProjectConstraints) "[gen] 造型 %s 有 3D 资产(%d bytes),走三渲二", input.outfit_id or "?", len(rigged), ) - generated = self._get_generator( - _resolve_video_model(input.video_model)).generate_rendered( + generated = self._get_generator().generate_rendered( card, action, rigged, progress, canvas=canvas ) else: master = (self._fetch_master or self._download_master)(input) - generated = self._get_generator( - _resolve_video_model(input.video_model)).generate( + generated = self._get_generator().generate( card, action, master, progress, canvas=canvas ) From ce2de8c99ddfbe2c7de66f602f6698631ad84013 Mon Sep 17 00:00:00 2001 From: xiaocheny214 <187097481+xiaocheny214@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:19:02 +0800 Subject: [PATCH 15/15] =?UTF-8?q?fix(gateway):=20=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=E5=90=88=E5=85=A5=E5=90=8E=E7=9A=84=20chat=20=E5=B7=A5?= =?UTF-8?q?=E5=8E=82=E4=B8=8E=E9=A2=84=E4=BB=98=E8=B4=B9=E7=BB=93=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat 工厂校验 AI_API_KEY / AI_CHAT_MODEL；图片失败解冻积分；stance 测试改打 generate_character_action。 --- .../server/orchestrator/executor.py | 2 + .../src/windup_framework/gateway/chat.py | 11 +++-- .../src/windup_framework/providers/chat.py | 4 ++ backend/tests/test_gateway_chat.py | 2 +- backend/tests/test_gateway_executor.py | 9 ++++- backend/tests/test_generation_api.py | 40 ++++++++++--------- 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/backend/packages/app/src/windup_app/server/orchestrator/executor.py b/backend/packages/app/src/windup_app/server/orchestrator/executor.py index 1c585e8f..8f8ce60b 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/executor.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/executor.py @@ -525,10 +525,12 @@ def run_image_task( session.commit() except Exception as exc: # noqa: BLE001 —— 兜底 logger.exception("图片任务 %s 失败", task_id) + session.rollback() task_repo.update_status( session, task_id, TaskStatus.FAILED, error_message=f"{exc}; request_id={request_id}", ) + _settle_credit(session, task_id, success=False) if own: session.commit() finally: diff --git a/backend/packages/framework/src/windup_framework/gateway/chat.py b/backend/packages/framework/src/windup_framework/gateway/chat.py index ccb95888..7ba4a714 100644 --- a/backend/packages/framework/src/windup_framework/gateway/chat.py +++ b/backend/packages/framework/src/windup_framework/gateway/chat.py @@ -111,10 +111,15 @@ def __init__(self, adapter, circuit, settings, route_adapters=None) -> None: def _adapter_for(self, route: GatewayRoute): return lookup_adapter(self._route_adapters, route, self._adapter) + @property + def model_name(self) -> str: + return (self._settings.chat_model or self._settings.model).strip() + def _models(self) -> tuple[str, ...]: - if not self._settings.model.strip(): - raise RuntimeError("chat gateway requires AI_MODEL") - return (self._settings.model, *_parse_fallbacks(self._settings.chat_fallbacks)) + primary = self.model_name + if not primary: + raise RuntimeError("chat gateway requires AI_CHAT_MODEL") + return (primary, *_parse_fallbacks(self._settings.chat_fallbacks)) def invoke(self, messages: Any, **kwargs: Any) -> Any: ctx = current_call_context() diff --git a/backend/packages/framework/src/windup_framework/providers/chat.py b/backend/packages/framework/src/windup_framework/providers/chat.py index 606d816f..13669900 100644 --- a/backend/packages/framework/src/windup_framework/providers/chat.py +++ b/backend/packages/framework/src/windup_framework/providers/chat.py @@ -15,4 +15,8 @@ def create_chat_model( 协议适配仍由 LangChain 官方 ``ChatOpenAI`` 完成；Gateway 只负责 route / retry / circuit / trace。 """ + if not config.api_key.strip(): + raise ValueError("AI_API_KEY is required") + if not (config.chat_model or config.model).strip(): + raise ValueError("AI_CHAT_MODEL is required") return build_chat_gateway(config=config, **kwargs) diff --git a/backend/tests/test_gateway_chat.py b/backend/tests/test_gateway_chat.py index 94340f6e..a6f218a6 100644 --- a/backend/tests/test_gateway_chat.py +++ b/backend/tests/test_gateway_chat.py @@ -88,7 +88,7 @@ def test_chat_gateway_switches_key_after_429(monkeypatch, caplog): def test_create_chat_model_returns_gateway_without_hand_rolling_protocol(): - cfg = AIProviderSettings(model="gpt-4o-mini") + cfg = AIProviderSettings(api_key="test-key", model="gpt-4o-mini") chat = create_chat_model(config=cfg) assert hasattr(chat, "invoke") diff --git a/backend/tests/test_gateway_executor.py b/backend/tests/test_gateway_executor.py index dc7e1132..7fd5ba8f 100644 --- a/backend/tests/test_gateway_executor.py +++ b/backend/tests/test_gateway_executor.py @@ -21,6 +21,9 @@ ) from windup_app.server.orchestrator.service import AiGenerationService from windup_app.server.project.model import Project # noqa: F401 — 注册表 +from windup_app.server.quota.model import CreditAccount, CreditTransaction # noqa: F401 — 注册表 + +from conftest import seed_credit_account @pytest.fixture @@ -31,7 +34,11 @@ def session_factory(): poolclass=StaticPool, ) Base.metadata.create_all(engine) - return sessionmaker(bind=engine) + factory = sessionmaker(bind=engine) + with factory() as session: + seed_credit_account(session, 1) + session.commit() + return factory def test_none_video_model_means_deploy_default(): diff --git a/backend/tests/test_generation_api.py b/backend/tests/test_generation_api.py index 6953e17c..56af8bc4 100644 --- a/backend/tests/test_generation_api.py +++ b/backend/tests/test_generation_api.py @@ -255,13 +255,16 @@ def test_validation_error_message_tells_the_user_what_is_wrong(auth_client): def test_stance_from_request_reaches_the_engine(auth_client, monkeypatch): - from windup_app.web.api import generation as gen_api - from windup_app.server.orchestrator.model import CharacterActionInput + from windup_app.server.orchestrator import service as gen_service - dispatched: list = [] - monkeypatch.setattr( - gen_api, "_dispatch_after_commit", lambda *args: dispatched.append(args) - ) + captured: list = [] + _orig = gen_service.service.generate_character_action + + def _spy(*args, **kwargs): + captured.append(kwargs.get("input")) + return _orig(*args, **kwargs) + + monkeypatch.setattr(gen_service.service, "generate_character_action", _spy) project = _create_project(auth_client) character = _create_character(auth_client, project["id"]) @@ -270,21 +273,23 @@ def test_stance_from_request_reaches_the_engine(auth_client, monkeypatch): json=_action_payload(project["id"], character["id"], stance="quadruped"), ) - inputs = [a for args in dispatched for a in args if isinstance(a, CharacterActionInput)] - assert inputs, "任务没被收下" - assert inputs[0].stance is not None, "体型断在请求层,引擎侧永远看不到" - assert inputs[0].stance.value == "quadruped" + assert captured, "任务没被收下" + assert captured[0].stance is not None, "体型断在请求层,引擎侧永远看不到" + assert captured[0].stance.value == "quadruped" def test_stance_omitted_stays_none_not_biped(auth_client, monkeypatch): """不给体型时原样传 None —— 在这层替调用方填 biped，"没给"与"明确双足"就分不开了。""" - from windup_app.web.api import generation as gen_api - from windup_app.server.orchestrator.model import CharacterActionInput + from windup_app.server.orchestrator import service as gen_service - dispatched: list = [] - monkeypatch.setattr( - gen_api, "_dispatch_after_commit", lambda *args: dispatched.append(args) - ) + captured: list = [] + _orig = gen_service.service.generate_character_action + + def _spy(*args, **kwargs): + captured.append(kwargs.get("input")) + return _orig(*args, **kwargs) + + monkeypatch.setattr(gen_service.service, "generate_character_action", _spy) project = _create_project(auth_client) character = _create_character(auth_client, project["id"]) @@ -292,8 +297,7 @@ def test_stance_omitted_stays_none_not_biped(auth_client, monkeypatch): "/generation/action", json=_action_payload(project["id"], character["id"]), ) - inputs = [a for args in dispatched for a in args if isinstance(a, CharacterActionInput)] - assert inputs and inputs[0].stance is None + assert captured and captured[0].stance is None def test_illegal_stance_is_rejected_at_the_entrance(auth_client):