From b636129e04e0f1dd1666d47c32d93061b8c1cb52 Mon Sep 17 00:00:00 2001 From: Abhyudaya Baiju Tharakan Date: Fri, 9 Oct 2026 06:46:44 +0530 Subject: [PATCH] feat(providers): add opt-in local inference provider adapters Register ollama, vllm, lmstudio, llamacpp and sglang as OpenAI-compatible providers only when their base URL variable is set (all unset by default). URLs are validated to an explicit /v1 endpoint, plain HTTP is limited to loopback and private addresses, and an optional provider key is used when configured; an explicitly configured local server may be called without one. No port scanning, discovery or downloads; catalog entries carry declared capabilities with provenance. --- config.py | 2 + docs/local-inference.md | 76 ++++++ providers/local_inference.py | 129 ++++++++++ providers/registry.py | 2 + routes/provider_credentials.py | 6 +- services/auth_service.py | 18 ++ services/model_registry.py | 2 + services/provider_catalog_service.py | 26 +++ tests/test_local_inference.py | 338 +++++++++++++++++++++++++++ worker/container-env.mjs | 10 + 10 files changed, 608 insertions(+), 1 deletion(-) create mode 100644 docs/local-inference.md create mode 100644 providers/local_inference.py create mode 100644 tests/test_local_inference.py diff --git a/config.py b/config.py index fc46f924..52b668ef 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ from providers.cline_pass import CLINE_API_BASE_URL from providers.codex_everywhere import codex_everywhere_base_urls from providers.image_relays import image_relay_base_urls +from providers.local_inference import local_inference_base_urls from providers.nanogpt import nanogpt_text_base_url @@ -212,6 +213,7 @@ class Config: 'gemma': 'https://generativelanguage.googleapis.com/v1beta', **codex_everywhere_base_urls(), **image_relay_base_urls(), + **local_inference_base_urls(), } # Provider-specific timeouts (connect_timeout, read_timeout) diff --git a/docs/local-inference.md b/docs/local-inference.md new file mode 100644 index 00000000..6629a160 --- /dev/null +++ b/docs/local-inference.md @@ -0,0 +1,76 @@ +# Local inference providers + +Configure an OpenAI-compatible server explicitly to use `ollama`, `vllm`, +`lmstudio`, `llamacpp`, or `sglang` model prefixes. All five providers are +disabled when their base URL variables are unset or empty. There is no port +scanning, host discovery, model download, or inference runtime installation. + +| Provider | Base URL variable | Optional provider key variable | +| --- | --- | --- | +| `ollama` | `OLLAMA_BASE_URL` | `OLLAMA_API_KEY` | +| `vllm` | `VLLM_BASE_URL` | `VLLM_API_KEY` | +| `lmstudio` | `LM_STUDIO_BASE_URL` | `LM_STUDIO_API_KEY` | +| `llamacpp` | `LLAMA_CPP_BASE_URL` | `LLAMA_CPP_API_KEY` | +| `sglang` | `SGLANG_BASE_URL` | `SGLANG_API_KEY` | + +For example, set `OLLAMA_BASE_URL=http://127.0.0.1:11434/v1` before starting +standalone Flask. A bare origin receives `/v1`; a trailing slash is removed. +A service prefix such as `https://inference.example/serve/v1` is preserved. +Configure the server's actual OpenAI-compatible endpoint; Ollama's native +`/api/chat` protocol is not supported here. + +URL validation rejects embedded credentials, query strings, fragments, +encoded paths, path traversal, invalid ports, and paths outside `/v1`. +Malformed settings disable that provider and log the variable name once without +its value. Plain HTTP accepts only explicitly configured loopback, RFC 1918, +or IPv6 unique-local addresses, and `localhost` names. Public HTTP, wildcard +bind addresses, multicast, and link-local metadata addresses are rejected. +Private DNS names require HTTPS; validation does not resolve DNS names. + +Use a model already loaded on your server: + +```json +{ + "model": "ollama:example-model:latest", + "messages": [{"role": "user", "content": "Hello"}] +} +``` + +Send this body to `/v1/chat/completions` with your gateway authorization. +Only the provider prefix is removed from the upstream model ID. Requests go to +the configured `/v1/chat/completions`. Managed `/v1/responses` uses the existing +Chat conversion; native Responses continuation controls such as +`previous_response_id` return a real validation error before dispatch. +Gateway authorization remains required even when the local server needs no key. +Provider keys use the private provider credential loader and are sent only to +the configured upstream, never copied from the caller's gateway authorization. + +A provider key is optional. When none is configured, an explicitly configured +local server is called without an `Authorization` header, both for generation +and for catalog refresh. Providers other than these five still refuse a request +when their key is missing. + +Catalog refresh queries only configured servers at `/v1/models` using existing +refresh controls and transport timeouts. Results appear in `/v1/models` with +provider prefixes. Explicit model IDs do not require catalog discovery. +Failed refreshes retain the last good catalog and report the real failure; +disabled local providers' saved catalog entries are hidden. +Catalogs retain bounded provider declarations and attach +`provider_catalog:` provenance. Tools, vision, and other optional +capabilities are not inferred from model names or assumed from OpenAI API +compatibility. The adapter declares Chat and streaming transport support only. +The existing precision policy exposes declared precision when +`MODEL_PRECISION_PREFERENCE` is enabled; missing precision stays `unknown`. +Compatibility does not certify a server's model capabilities. + +Local inference behind the Worker runs through Flask in its Container. Loopback +there means the Container, not the operator's workstation. Use a configured +origin reachable from the Container and follow the existing outbound/SSRF +policy; configuring a URL does not create connectivity or bypass that policy. +There is no GPU inference on the Worker and no additional native Worker route. + +No monetary price is inferred for local compute. Existing accounting and +retention policies still apply; requests are not free merely because a server +is local. This integration stores model catalog metadata without provider +credentials and adds no prompt or response logging. Upstream errors remain +errors, and ambiguous generation failures do not authorize a new retry. diff --git a/providers/local_inference.py b/providers/local_inference.py new file mode 100644 index 00000000..5fb4b1ce --- /dev/null +++ b/providers/local_inference.py @@ -0,0 +1,129 @@ +"""Explicit OpenAI-compatible local endpoints; no host or model discovery.""" + +from collections.abc import Mapping +import ipaddress +import logging +import os +import re +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from providers.base import ProviderCapabilities +from providers.openai_compatible import OpenAICompatibleAdapter + +LOCAL_INFERENCE_ENV = { + "ollama": ("OLLAMA_BASE_URL", "OLLAMA_API_KEY"), + "vllm": ("VLLM_BASE_URL", "VLLM_API_KEY"), + "lmstudio": ("LM_STUDIO_BASE_URL", "LM_STUDIO_API_KEY"), + "llamacpp": ("LLAMA_CPP_BASE_URL", "LLAMA_CPP_API_KEY"), + "sglang": ("SGLANG_BASE_URL", "SGLANG_API_KEY"), +} +LOCAL_INFERENCE_PROVIDERS = frozenset(LOCAL_INFERENCE_ENV) +_PRIVATE_NETWORKS = tuple(ipaddress.ip_network(value) for value in ( + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", +)) +_HOST_PATTERN = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", re.IGNORECASE) +_PATH_PATTERN = re.compile(r"(?:/[A-Za-z0-9_-]+)*/v1") +_invalid_settings_logged: set[str] = set() +logger = logging.getLogger(__name__) + + +def normalize_local_base_url(value: str) -> str: + """Validate an operator endpoint without resolving names or probing ports.""" + if len(value) > 2048 or any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Invalid local inference base URL") + candidate = value.strip() + if any(char in candidate for char in "\\%?#"): + raise ValueError("Invalid local inference base URL") + parsed = urlsplit(candidate) + host = (parsed.hostname or "").lower().rstrip(".") + port = parsed.port + if (parsed.scheme not in {"http", "https"} or not host + or parsed.username is not None or parsed.password is not None + or (port is not None and not 1 <= port <= 65535)): + raise ValueError("Invalid local inference origin") + try: + address = ipaddress.ip_address(host) + except ValueError: + address = None + private = bool(address and (address.is_loopback or any( + address.version == network.version and address in network + for network in _PRIVATE_NETWORKS + ))) + localhost = host == "localhost" or host.endswith(".localhost") + if address is not None and not (private or address.is_global): + raise ValueError("Invalid local inference address") + if address is None and not _HOST_PATTERN.fullmatch(host): + raise ValueError("Invalid local inference hostname") + if parsed.scheme == "http" and not (private or localhost): + raise ValueError("Local HTTP requires a loopback or private address") + path = parsed.path.rstrip("/") or "/v1" + if not _PATH_PATTERN.fullmatch(path): + raise ValueError("Local inference requires an OpenAI-compatible /v1 service") + authority = f"[{host}]" if ":" in host else host + if port is not None: + authority += f":{port}" + return urlunsplit((parsed.scheme, authority, path, "", "")) + + +def local_inference_base_urls() -> dict[str, str]: + urls = {} + for provider, (env_name, _) in LOCAL_INFERENCE_ENV.items(): + value = os.environ.get(env_name, "") + if not value.strip(): + continue + try: + urls[provider] = normalize_local_base_url(value) + except ValueError: + if env_name not in _invalid_settings_logged: + _invalid_settings_logged.add(env_name) + logger.warning("Ignoring invalid %s; local inference provider is off", env_name) + return urls + + +def local_inference_adapters(base_urls: Mapping[str, str]) -> dict[str, OpenAICompatibleAdapter]: + adapters = {} + for provider in LOCAL_INFERENCE_ENV: + value = base_urls.get(provider) + if not value: + continue + try: + base_url = normalize_local_base_url(value) + except ValueError: + continue + adapters[provider] = OpenAICompatibleAdapter( + name=provider, base_url=base_url, chat_path="chat/completions", + provider_capabilities=ProviderCapabilities(), + ) + return adapters + + +def local_credential_env_names(provider: str) -> tuple[str, ...]: + settings = LOCAL_INFERENCE_ENV.get(provider) + return (settings[1],) if settings else () + + +def local_provider_allows_keyless(provider: str, base_urls: Mapping[str, str]) -> bool: + return provider in local_inference_adapters(base_urls) + + +def local_catalog_metadata(provider: str, metadata: Mapping[str, Any] | None) -> dict[str, Any]: + """Tag declarations, leaving model-name guesses and monetary cost unknown.""" + result = dict(metadata or {}) + source = f"provider_catalog:{provider}" + fields = {key for key in result if key.startswith("supports_") or key in { + "input_modalities", "output_modalities", "context_window", "max_output_tokens", + }} + result["metadata_source"] = source + result["metadata_provenance"] = { + **(result.get("metadata_provenance") or {}), + **{key: source for key in fields}, + } + # Precision declarations follow the existing operator precision policy. + from services.model_precision import configured_preference + + if configured_preference(): + result.setdefault("precision", "unknown") + result.setdefault("precision_source", source) + result["metadata_provenance"]["precision"] = result["precision_source"] + return result diff --git a/providers/registry.py b/providers/registry.py index 159323a6..290f04d9 100644 --- a/providers/registry.py +++ b/providers/registry.py @@ -6,6 +6,7 @@ from providers.base import ProviderCapabilities from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS, OPENAI_PROTOCOL from providers.image_relays import image_relay_specs +from providers.local_inference import local_inference_adapters from providers.openai_compatible import OpenAICompatibleAdapter @@ -174,6 +175,7 @@ def build_default_registry(base_urls: Mapping[str, str]) -> dict[str, OpenAIComp chat_path="v1/chat/completions", provider_capabilities=spec.capabilities(), ) + registry.update(local_inference_adapters(base_urls)) return registry diff --git a/routes/provider_credentials.py b/routes/provider_credentials.py index d45c3439..9c7e3c3a 100644 --- a/routes/provider_credentials.py +++ b/routes/provider_credentials.py @@ -15,7 +15,7 @@ from services.nanogpt_key_pool import NanoGPTUnifiedKeyPool as NanoGPTKeyPool -def _provider_token(app, auth_service_cls, proxy_service_cls, provider: str, *, model=None, quota_bucket=None) -> str: +def _provider_token(app, auth_service_cls, proxy_service_cls, provider: str, *, model=None, quota_bucket=None) -> str | None: context = selection_context(provider, model, config=app.config, quota_bucket=quota_bucket) if provider == "googleai": token = auth_service_cls.get_google_token() @@ -42,6 +42,10 @@ def _provider_token(app, auth_service_cls, proxy_service_cls, provider: str, *, else: token = auth_service_cls.get_api_key(provider, **context) if not token: + # Only an explicitly configured local inference server may be called without a key. + requires_key = getattr(auth_service_cls, "provider_requires_api_key", None) + if requires_key is not None and not requires_key(provider, app.config.get("API_BASE_URLS", {})): + return None raise AutoRouteCandidateUnavailable( f"API key not configured for {provider}", ) diff --git a/services/auth_service.py b/services/auth_service.py index 4c2c593f..07e6a06a 100644 --- a/services/auth_service.py +++ b/services/auth_service.py @@ -22,6 +22,12 @@ from error_handlers import APIError from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS from providers.image_relays import image_relay_api_key, image_relay_specs +from providers.local_inference import ( + LOCAL_INFERENCE_PROVIDERS, + local_credential_env_names, + local_inference_base_urls, + local_provider_allows_keyless, +) from services.credential_pool import CredentialPool from services.auth_primitives import ( DEFAULT_ADMIN_SCOPES, @@ -672,6 +678,11 @@ def initialize(cls) -> None: @classmethod def get_api_key(cls, provider: str, *, model: str | None = None, quota_bucket: str | None = None) -> Optional[str]: """Get API key for a provider.""" + if provider in LOCAL_INFERENCE_PROVIDERS: + if provider not in local_inference_base_urls(): + return None + return next((key for name in local_credential_env_names(provider) + if (key := usable_credential(os.environ.get(name), name))), None) if CredentialPool.pooled(provider): pooled = CredentialPool.select(provider, model=model, quota_bucket=quota_bucket) if pooled: @@ -688,8 +699,15 @@ def get_api_key(cls, provider: str, *, model: str | None = None, quota_bucket: s @classmethod def provider_credential_env_names(cls, provider: str) -> tuple[str, ...]: """Return safe environment-variable names for dashboard setup guidance.""" + if provider in LOCAL_INFERENCE_PROVIDERS: + return local_credential_env_names(provider) return provider_credential_env_names(provider) + @staticmethod + def provider_requires_api_key(provider: str, base_urls) -> bool: + """Local servers may be keyless only for an explicitly configured endpoint.""" + return not local_provider_allows_keyless(provider, base_urls) + @classmethod def get_api_keys(cls, provider: str) -> List[str]: """Get every configured key for providers that support a key pool.""" diff --git a/services/model_registry.py b/services/model_registry.py index 554383bd..2a2ffa32 100644 --- a/services/model_registry.py +++ b/services/model_registry.py @@ -12,6 +12,7 @@ from providers.cline_pass import CLINE_PASS_MODEL_IDS from providers.codex_everywhere import CODEX_EVERYWHERE_PROVIDERS from providers.image_relays import image_relay_model_ids +from providers.local_inference import LOCAL_INFERENCE_PROVIDERS from providers.opencode_go import ( OPENCODE_GO_LEGACY_MODEL_IDS, OPENCODE_GO_MODEL_IDS, @@ -31,6 +32,7 @@ } DYNAMIC_PROVIDER_MODELS = { + *LOCAL_INFERENCE_PROVIDERS, "codex-easy", *CODEX_EVERYWHERE_PROVIDERS, "linkapi", diff --git a/services/provider_catalog_service.py b/services/provider_catalog_service.py index c6f96574..a755f1ec 100644 --- a/services/provider_catalog_service.py +++ b/services/provider_catalog_service.py @@ -15,6 +15,12 @@ ) from providers.codex_everywhere import CODEX_EVERYWHERE_POOLS from providers.image_relays import image_relay_specs +from providers.local_inference import ( + LOCAL_INFERENCE_PROVIDERS, + local_catalog_metadata, + local_inference_base_urls, + normalize_local_base_url, +) from providers.opencode_go import is_opencode_zen_free_model from services import provider_catalog_d1 from services.provider_capability_discovery import enrich_model_capabilities @@ -48,6 +54,8 @@ class ProviderCatalogModel: PROVIDER_CATALOG_SPECS = { + **{provider: ProviderCatalogSpec("models", f"/{provider}/models") + for provider in sorted(LOCAL_INFERENCE_PROVIDERS)}, "aihubmix": ProviderCatalogSpec("v1/models", "/aihubmix/v1/models"), "openai": ProviderCatalogSpec("v1/models", "/openai/v1/models"), "openrouter": ProviderCatalogSpec("models", "/openrouter/models"), @@ -175,6 +183,13 @@ def _ensure_storage(cls, connection: sqlite3.Connection) -> None: @classmethod def list_models(cls) -> list[ProviderCatalogModel]: + models = cls._stored_models() + enabled = local_inference_base_urls() + return [model for model in models + if model.provider not in LOCAL_INFERENCE_PROVIDERS or model.provider in enabled] + + @classmethod + def _stored_models(cls) -> list[ProviderCatalogModel]: if provider_catalog_d1.using_d1(): return list(cls._durable_models()) cache_key = cls._cache_key() @@ -219,6 +234,8 @@ def list_models(cls) -> list[ProviderCatalogModel]: @classmethod def has_model(cls, provider: str, model_id: str) -> bool: """Return whether a model appears in the last successful live catalog.""" + if provider in LOCAL_INFERENCE_PROVIDERS and provider not in local_inference_base_urls(): + return False if provider_catalog_d1.using_d1(): return any( model.provider == provider and model.model_id == model_id @@ -408,6 +425,8 @@ def extract_models( ) existing = normalized.get(model_id) metadata = sanitize_provider_metadata(item) + if provider in LOCAL_INFERENCE_PROVIDERS: + metadata = local_catalog_metadata(provider, metadata) if existing: context_window = cls._minimum_limit( existing.context_window, @@ -440,6 +459,8 @@ def _credential_candidates( auth_service_cls, provider: str, ) -> tuple[str | None, ...]: + if provider in LOCAL_INFERENCE_PROVIDERS: + return (auth_service_cls.get_api_key(provider),) if provider in PUBLIC_CATALOG_PROVIDERS: return (None,) if provider == "nanogpt": @@ -599,6 +620,11 @@ def refresh_configured( base_url = base_urls.get(provider) if not base_url: continue + if provider in LOCAL_INFERENCE_PROVIDERS: + try: + base_url = normalize_local_base_url(base_url) + except ValueError: + continue credentials = cls._credential_candidates(auth_service_cls, provider) if not credentials: results.append( diff --git a/tests/test_local_inference.py b/tests/test_local_inference.py new file mode 100644 index 00000000..2754ab32 --- /dev/null +++ b/tests/test_local_inference.py @@ -0,0 +1,338 @@ +"""Explicit local endpoints reuse the managed OpenAI-compatible transport.""" +import importlib +import json +import os +from unittest.mock import patch + +import pytest + +from tests.unified_api_test_case import UnifiedApiTestCase + +URL_KEYS = { + "ollama": "OLLAMA_BASE_URL", "vllm": "VLLM_BASE_URL", + "lmstudio": "LM_STUDIO_BASE_URL", "llamacpp": "LLAMA_CPP_BASE_URL", + "sglang": "SGLANG_BASE_URL", +} +KEY_KEYS = {"ollama": "OLLAMA_API_KEY", "vllm": "VLLM_API_KEY", + "lmstudio": "LM_STUDIO_API_KEY", "llamacpp": "LLAMA_CPP_API_KEY", + "sglang": "SGLANG_API_KEY"} + + +@pytest.fixture +def isolated(monkeypatch, tmp_path): + import config + import env_loader + monkeypatch.setattr(config, "load_runtime_env", lambda: None) + monkeypatch.setattr(env_loader, "load_runtime_env", lambda: None) + for name in (*URL_KEYS.values(), *KEY_KEYS.values(), + "LMSTUDIO_API_KEY", "LLAMACPP_API_KEY"): + monkeypatch.setenv(name, "") + monkeypatch.setenv("MODEL_REGISTRY_DB_PATH", str(tmp_path / "models.sqlite3")) + monkeypatch.setenv("CONTROL_PLANE_DATABASE_URL", "") + monkeypatch.setenv("CONTROL_STATE_BACKEND", "local") + monkeypatch.setenv("MODEL_PRECISION_PREFERENCE", "") + from services.auth_service import AuthService + from services.credential_pool import CredentialPool + monkeypatch.setattr(AuthService, "_api_keys", {}) + monkeypatch.setattr(CredentialPool, "pooled", lambda provider: False) + return importlib.import_module("providers.local_inference") + + +@pytest.mark.parametrize("provider", URL_KEYS) +def test_only_explicit_urls_register(provider, isolated, monkeypatch): + from providers.registry import build_default_registry + assert isolated.local_inference_base_urls() == {} + monkeypatch.setenv(URL_KEYS[provider], "http://127.0.0.1:8000/v1/") + urls = isolated.local_inference_base_urls() + assert urls == {provider: "http://127.0.0.1:8000/v1"} + adapter = build_default_registry(urls)[provider] + assert adapter.chat_completions_url() == "http://127.0.0.1:8000/v1/chat/completions" + assert adapter.capabilities().supports_chat + assert not adapter.capabilities().supports_tools + assert not adapter.capabilities().supports_images + + +@pytest.mark.parametrize("value,expected", [ + ("http://localhost:11434", "http://localhost:11434/v1"), + ("http://10.1.2.3:8000/v1/", "http://10.1.2.3:8000/v1"), + ("http://172.16.0.1/v1", "http://172.16.0.1/v1"), + ("http://192.168.1.3/v1", "http://192.168.1.3/v1"), + ("http://[::1]:8080/v1", "http://[::1]:8080/v1"), + ("http://[fd00::1]/v1", "http://[fd00::1]/v1"), + ("https://inference.example/serve/v1/", "https://inference.example/serve/v1"), +]) +def test_v1_normalization_and_private_http(value, expected, isolated): + assert isolated.normalize_local_base_url(value) == expected + + +@pytest.mark.parametrize("value", [ + "http://public.example/v1", "http://8.8.8.8/v1", "http://169.254.169.254/v1", + "http://0.0.0.0/v1", "http://[::]/v1", "http://224.0.0.1/v1", + "http://127.0.0.1:0/v1", "http://localhost:99999/v1", + "https://user:secret@inference.example/v1", "file:///v1", "/v1", + "http://localhost/api/chat", "http://localhost/v1?key=secret", + "http://localhost/v1#fragment", "http://localhost/a/../v1", + "http://localhost/%2e%2e/v1", "http://localhost\\evil/v1", + "http://localhost/v1\n", "https://bad host/v1", +]) +def test_malformed_origins_disable_without_logging_values(value, isolated, monkeypatch, caplog): + isolated._invalid_settings_logged.clear() + monkeypatch.setenv("OLLAMA_BASE_URL", value) + assert isolated.local_inference_base_urls() == {} + assert isolated.local_inference_base_urls() == {} + assert len(caplog.records) == 1 + assert "OLLAMA_BASE_URL" in caplog.text + assert value not in caplog.text + + +def test_disabled_configuration_and_adapter_output_are_unchanged(isolated, monkeypatch): + from providers.base import CanonicalRequest + from providers.registry import build_default_registry + from config import Config + with patch("env_loader.load_runtime_env"), patch("config.load_runtime_env"): + module = importlib.reload(importlib.import_module("config")) + assert not set(URL_KEYS).intersection(module.Config.API_BASE_URLS) + original = {"openai": "https://api.openai.com"} + registry = build_default_registry(original) + assert set(registry) == {"openai"} + payload = {"model": "gpt-4.1", "messages": [], "temperature": 0.4} + request = registry["openai"].prepare_request(CanonicalRequest("openai", raw=payload)) + assert request.data == json.dumps(payload).encode() + assert request.url == "https://api.openai.com/v1/chat/completions" + monkeypatch.setattr(Config, "API_BASE_URLS", original) + + +@pytest.mark.parametrize("provider", URL_KEYS) +def test_private_key_mapping_and_optional_auth(provider, isolated, monkeypatch): + from services.auth_service import AuthService + monkeypatch.setenv(URL_KEYS[provider], "http://localhost:8000/v1") + urls = isolated.local_inference_base_urls() + assert AuthService.get_api_key(provider) is None + assert not AuthService.provider_requires_api_key(provider, urls) + assert AuthService.provider_requires_api_key(provider, {}) + assert AuthService.provider_requires_api_key("openai", urls) + assert AuthService.provider_credential_env_names(provider)[0] == KEY_KEYS[provider] + monkeypatch.setenv(KEY_KEYS[provider], "synthetic-local-key") + assert AuthService.get_api_key(provider) == "synthetic-local-key" + monkeypatch.setenv(KEY_KEYS[provider], "your-local-api-key") + assert AuthService.get_api_key(provider) is None + monkeypatch.setenv(URL_KEYS[provider], "") + assert AuthService.get_api_key(provider) is None + + +def test_catalog_refresh_is_explicit_and_keyless(isolated, monkeypatch): + from services.auth_service import AuthService + from services.provider_catalog_service import ProviderCatalogService + from services.proxy_service import ProxyService + monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") + response = UnifiedApiTestCase._chat_response() + response._content = json.dumps({"data": [{"id": "model:latest"}]}).encode() + with patch.object(ProxyService, "make_request", return_value=response) as transport: + results = ProviderCatalogService.refresh_configured( + isolated.local_inference_base_urls(), AuthService, ProxyService) + assert results == [{"provider": "ollama", "status": "updated", "truncated": False, + "model_count": 1}] + assert transport.call_count == 1 + kwargs = transport.call_args.kwargs + assert kwargs["url"] == "http://localhost:11434/v1/models" + assert "Authorization" not in kwargs["headers"] + assert kwargs["force_raw_passthrough"] + assert [model.model_id for model in ProviderCatalogService.list_models()] == ["model:latest"] + monkeypatch.setenv("OLLAMA_BASE_URL", "") + assert ProviderCatalogService.list_models() == [] + with patch.object(ProxyService, "make_request") as transport: + assert ProviderCatalogService.refresh_configured({}, AuthService, ProxyService) == [] + transport.assert_not_called() + + +def test_catalog_proxy_paths_match_normalized_v1_base(isolated): + from services.provider_catalog_service import PROVIDER_CATALOG_SPECS + for provider in URL_KEYS: + spec = PROVIDER_CATALOG_SPECS[provider] + assert spec.upstream_path == "models" + assert spec.proxy_path == f"/{provider}/models" + + +def test_catalog_keeps_only_declarations_with_provenance(isolated, monkeypatch): + from services.provider_catalog_service import ProviderCatalogService + monkeypatch.setenv("MODEL_PRECISION_PREFERENCE", '["fp16"]') + records = ProviderCatalogService.extract_models("vllm", {"data": [ + {"id": "some-int4", "supports_tools": True, "precision": "fp16"}, + {"id": "vision-int4", "api_key": "synthetic-secret"}, + ]}) + known, unknown = [record.metadata for record in records] + assert known["supports_tools"] is True + assert known["precision"] == "fp16" + assert known["metadata_provenance"]["supports_tools"] == "provider_catalog:vllm" + assert known["precision_source"] == "provider_catalog:vllm" + assert unknown["precision"] == "unknown" + assert "supports_vision" not in unknown + assert "api_key" not in unknown + + +def test_explicit_models_resolve_without_discovery(isolated, monkeypatch): + from services.model_registry import ModelRegistry + monkeypatch.setenv("VLLM_BASE_URL", "http://localhost:8000/v1") + assert ModelRegistry.get_model("vllm:org/unlisted:latest", isolated.local_inference_base_urls()) + assert ModelRegistry.get_model("vllm:org/unlisted:latest", {}) is None + + +def test_config_registers_all_five_urls_only_when_set(isolated, monkeypatch): + config = importlib.import_module("config") + original = config.Config + for name in URL_KEYS.values(): + monkeypatch.setenv(name, "http://127.0.0.1:8000/v1/") + with patch("env_loader.load_runtime_env", lambda: None): + importlib.reload(config) + try: + assert set(URL_KEYS).issubset(config.Config.API_BASE_URLS) + assert all(config.Config.API_BASE_URLS[name] == "http://127.0.0.1:8000/v1" + for name in URL_KEYS) + finally: + config.Config = original + + +def test_invalid_direct_base_url_never_dispatches_catalog(isolated): + from providers.registry import build_default_registry + from services.provider_catalog_service import ProviderCatalogService + from services.auth_service import AuthService + from services.proxy_service import ProxyService + urls = {"ollama": "http://public.example/v1"} + assert build_default_registry(urls) == {} + with patch.object(ProxyService, "make_request") as transport: + assert ProviderCatalogService.refresh_configured(urls, AuthService, ProxyService) == [] + transport.assert_not_called() + + +def test_failed_catalog_retains_real_last_good_result(isolated, monkeypatch): + from services.provider_catalog_service import ProviderCatalogService + from services.auth_service import AuthService + from services.proxy_service import ProxyService + monkeypatch.setenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") + ProviderCatalogService.replace_provider_models("ollama", ("already-loaded",)) + response = UnifiedApiTestCase._chat_response() + response.status_code = 503 + response._content_consumed = True + with patch.object(ProxyService, "make_request", return_value=response) as transport: + results = ProviderCatalogService.refresh_configured( + isolated.local_inference_base_urls(), AuthService, ProxyService) + assert results[0]["status"] == "failed" + assert results[0]["message"] == "Catalog request returned HTTP 503" + assert transport.call_count == 1 + assert [model.model_id for model in ProviderCatalogService.list_models()] == ["already-loaded"] + + +class LocalInferenceRouteTest(UnifiedApiTestCase): + def setUp(self): + self.env_patch = patch.dict(os.environ, {**dict.fromkeys(URL_KEYS.values(), ""), + **dict.fromkeys(KEY_KEYS.values(), ""), "OLLAMA_BASE_URL": "http://localhost:11434/v1", + "OLLAMA_API_KEY": "synthetic-local-key", "PROVIDER_CATALOG_AUTO_REFRESH": "false"}) + self.env_patch.start() + self.addCleanup(self.env_patch.stop) + self.loader_patch = patch("env_loader.load_runtime_env", lambda: None) + self.loader_patch.start() + self.addCleanup(self.loader_patch.stop) + config_patch = patch("config.load_runtime_env", lambda: None) + config_patch.start() + self.addCleanup(config_patch.stop) + super().setUp() + self.app.config["API_BASE_URLS"] = {**self.app.config["API_BASE_URLS"], + "ollama": "http://localhost:11434/v1"} + + def test_chat_uses_configured_v1_and_original_model(self): + with patch.object(self.app_module.ProxyService, "make_request", + return_value=self._chat_response()) as transport: + response = self.client.post("/v1/chat/completions", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": "ollama:model:latest", "messages": [{"role": "user", "content": "hello"}]}) + self.assertEqual(response.status_code, 200, response.json) + self.assertEqual(transport.call_count, 1) + kwargs = transport.call_args.kwargs + self.assertEqual(kwargs["url"], "http://localhost:11434/v1/chat/completions") + self.assertEqual(json.loads(kwargs["data"])["model"], "model:latest") + self.assertEqual(kwargs["headers"]["Authorization"], "Bearer synthetic-local-key") + + def test_responses_translates_to_chat_and_rejects_native_controls(self): + with patch.object(self.app_module.ProxyService, "make_request", + return_value=self._chat_response()) as transport: + response = self.client.post("/v1/responses", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": "ollama:model:latest", "input": "hello"}) + self.assertEqual(response.status_code, 200, response.json) + self.assertEqual(transport.call_count, 1) + self.assertEqual(transport.call_args.kwargs["url"], "http://localhost:11434/v1/chat/completions") + with patch.object(self.app_module.ProxyService, "make_request") as transport: + response = self.client.post("/v1/responses", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": "ollama:model:latest", "input": "hello", "previous_response_id": "resp-native"}) + self.assertEqual(response.status_code, 400, response.json) + transport.assert_not_called() + + def test_models_expose_only_configured_catalog(self): + service = importlib.import_module("services.provider_catalog_service").ProviderCatalogService + service.replace_provider_models("ollama", service.extract_models("ollama", {"data": [{"id": "model:latest"}]})) + response = self.client.get("/v1/models", headers={"Authorization": "Bearer admin-test-key"}) + self.assertEqual(response.status_code, 200) + self.assertIn("ollama:model:latest", {row["id"] for row in response.json["data"]}) + + def test_disabled_provider_is_rejected_before_dispatch(self): + self.app.config["API_BASE_URLS"].pop("ollama") + with patch.object(self.app_module.ProxyService, "make_request") as transport: + response = self.client.post("/v1/chat/completions", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": "ollama:model:latest", "messages": []}) + self.assertEqual(response.status_code, 400) + transport.assert_not_called() + + def test_provider_failure_is_not_retried_or_replaced(self): + upstream = self._chat_response() + upstream.status_code = 503 + upstream._content = b'{"error":{"message":"local server unavailable"}}' + with patch.object(self.app_module.ProxyService, "make_request", + return_value=upstream) as transport: + response = self.client.post("/v1/chat/completions", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": "ollama:model:latest", "messages": []}) + self.assertEqual(response.status_code, 503) + self.assertEqual(transport.call_count, 1) + + def test_all_five_providers_use_managed_chat(self): + for provider in URL_KEYS: + with self.subTest(provider=provider): + self.app.config["API_BASE_URLS"][provider] = "http://localhost:8000/v1" + with patch.dict(os.environ, {URL_KEYS[provider]: "http://localhost:8000/v1", + KEY_KEYS[provider]: "synthetic-local-key"}), patch.object( + self.app_module.ProxyService, "make_request", + return_value=self._chat_response()) as transport: + response = self.client.post("/v1/chat/completions", + headers={"Authorization": "Bearer admin-test-key"}, + json={"model": f"{provider}:org/model:latest", "messages": []}) + self.assertEqual(response.status_code, 200, response.json) + self.assertEqual(transport.call_count, 1) + self.assertEqual(transport.call_args.kwargs["url"], + "http://localhost:8000/v1/chat/completions") + self.assertEqual(json.loads(transport.call_args.kwargs["data"])["model"], + "org/model:latest") + + def test_keyless_local_provider_dispatches_without_authorization(self): + auth = self.app_module.AuthService + self.assertFalse(auth.provider_requires_api_key("ollama", self.app.config["API_BASE_URLS"])) + for path, payload in ( + ("/v1/chat/completions", {"model": "ollama:model:latest", "messages": []}), + ("/v1/responses", {"model": "ollama:model:latest", "input": "hello"}), + ): + with self.subTest(path=path), patch.dict(os.environ, {"OLLAMA_API_KEY": ""}), patch.object( + self.app_module.ProxyService, "make_request", + return_value=self._chat_response()) as transport: + response = self.client.post(path, + headers={"Authorization": "Bearer admin-test-key"}, json=payload) + self.assertEqual(response.status_code, 200, response.json) + self.assertEqual(transport.call_count, 1) + self.assertNotIn("Authorization", transport.call_args.kwargs["headers"]) + + def test_keyed_provider_without_key_still_refuses(self): + credentials = importlib.import_module("routes.provider_credentials") + auth = self.app_module.AuthService + with patch.object(auth, "get_api_key", return_value=None): + with self.assertRaises(credentials.AutoRouteCandidateUnavailable): + credentials._provider_token(self.app, auth, self.app_module.ProxyService, "openai") diff --git a/worker/container-env.mjs b/worker/container-env.mjs index 1a7994db..0f64ccf6 100644 --- a/worker/container-env.mjs +++ b/worker/container-env.mjs @@ -40,6 +40,16 @@ const DIRECT_ENV_KEYS = [ "CONNECTION_PROFILES_DB_PATH", "CONFIG_REVISION_SYNC_ENABLED", "CONFIG_SYNC_TTL_SECONDS", + "LLAMA_CPP_API_KEY", + "LLAMA_CPP_BASE_URL", + "LM_STUDIO_API_KEY", + "LM_STUDIO_BASE_URL", + "OLLAMA_API_KEY", + "OLLAMA_BASE_URL", + "SGLANG_API_KEY", + "SGLANG_BASE_URL", + "VLLM_API_KEY", + "VLLM_BASE_URL", "GATEWAY_ALERTS_ENABLED", "GATEWAY_ALERT_WEBHOOK_ALLOWLIST", "CONFIG_SECURITY_TTL_SECONDS",