Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md

Large diffs are not rendered by default.

509 changes: 500 additions & 9 deletions dashboard/backend/api/routers/backtests.py

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions dashboard/backend/domain/analytics/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@
# field carries the reason. `provider_timeout` is a different fact with a
# different remedy.
"run_timeout",
# The account cannot take model usage at all (restricted) or cannot cover
# the next call's reservation. Both were being emitted already --
# `account_restricted` since the restricted lane shipped -- but neither
# was admitted here, so `emit_safe_error_event` raised inside its own
# guard and the event was dropped with only a WARNING line.
"account_restricted",
"credits_insufficient",
}

EVENT_GROUP_BY_NAME = {
Expand Down
3 changes: 2 additions & 1 deletion dashboard/backend/domain/analytics/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@
'credential_invalid', 'credential_missing', 'provider_timeout',
'provider_unavailable', 'provider_quota_exhausted',
'credits_unavailable',
'model_not_allowed', 'internal_error', 'run_timeout'
'model_not_allowed', 'internal_error', 'run_timeout',
'account_restricted', 'credits_insufficient'
)
),
country_code TEXT CHECK (country_code IS NULL OR length(country_code) = 2),
Expand Down
6 changes: 4 additions & 2 deletions dashboard/backend/domain/analytics/repository_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@
'credential_invalid', 'credential_missing', 'provider_timeout',
'provider_unavailable', 'provider_quota_exhausted',
'credits_unavailable',
'model_not_allowed', 'internal_error', 'run_timeout'
'model_not_allowed', 'internal_error', 'run_timeout',
'account_restricted', 'credits_insufficient'
)
),
country_code TEXT CHECK (country_code IS NULL OR length(country_code) = 2),
Expand Down Expand Up @@ -296,7 +297,8 @@ def _init_schema(self) -> None:
'provider_timeout', 'provider_unavailable',
'provider_quota_exhausted', 'credits_unavailable',
'model_not_allowed', 'internal_error',
'run_timeout'
'run_timeout', 'account_restricted',
'credits_insufficient'
)
)
"""
Expand Down
7 changes: 7 additions & 0 deletions dashboard/backend/domain/analytics/states.py
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,13 @@ def calculate_user_state(
"credential_missing",
"provider_unavailable",
"credits_unavailable",
# Split out of credits_unavailable; an empty balance still blocks the
# platform lane, so it keeps the operational state it always had.
"credits_insufficient",
# A restricted account (llm_overage, refund review) can run nothing on
# the platform lane -- strictly worse than an empty balance, so it
# must not read healthier than one.
"account_restricted",
"model_not_allowed",
}
resolvers = {
Expand Down
14 changes: 14 additions & 0 deletions dashboard/backend/domain/credits/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -1041,6 +1041,20 @@ def get_balance_projections(
def get_balance_micro(self, user_id: int) -> int:
return self.get_balance_projection(user_id)["total_available_micro"]

def has_promotion_grant(self, user_id: int, campaign_keys: Sequence[str]) -> bool:
"""Whether the account already holds a grant from any of these campaigns."""
keys = [str(key) for key in campaign_keys]
if not keys:
return False
placeholders = ", ".join("?" for _ in keys)
with self._get_connection() as conn:
row = conn.execute(
"SELECT 1 FROM credit_promotion_grants "
f"WHERE user_id = ? AND campaign_key IN ({placeholders}) LIMIT 1",
(int(user_id), *keys),
).fetchone()
return row is not None

def list_user_ids(self) -> list[int]:
"""Return account IDs for an idempotent promotion backfill."""
with self._get_connection() as conn:
Expand Down
15 changes: 15 additions & 0 deletions dashboard/backend/domain/credits/repository_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -895,6 +895,21 @@ def get_balance_projections(
def get_balance_micro(self, user_id: int) -> int:
return self.get_balance_projection(user_id)["total_available_micro"]

def has_promotion_grant(self, user_id: int, campaign_keys: Sequence[str]) -> bool:
"""Whether the account already holds a grant from any of these campaigns."""
keys = [str(key) for key in campaign_keys]
if not keys:
return False
with self._get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM credit_promotion_grants "
"WHERE user_id = %s AND campaign_key = ANY(%s) LIMIT 1",
(int(user_id), keys),
)
row = cur.fetchone()
return row is not None

def list_user_ids(self) -> list[int]:
"""Return account IDs for an idempotent promotion backfill."""
with self._get_connection() as conn:
Expand Down
18 changes: 16 additions & 2 deletions dashboard/backend/domain/credits/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,18 @@
from dashboard.backend.domain.analytics import instrumentation as analytics_instrumentation


DEFAULT_SIGNUP_CREDIT_CAMPAIGN = "default_signup_credits_v1"
DEFAULT_SIGNUP_CREDITS_MICRO = 1_500_000
# v2 (8 Credits) replaced v1 (1.5 Credits) on 2026-10-07: 1.5 could not pay for
# one default GPT-5.5 DJIA 30 backtest (~1.8 Credits), so a new account's first
# run on the default agent died partway. A new key rather than a new amount
# under the old one: the grant is idempotent per (campaign, user) and the
# amount is part of its digest, so changing it in place would raise an
# idempotency conflict for every existing account at each login and boot.
DEFAULT_SIGNUP_CREDIT_CAMPAIGN = "default_signup_credits_v2"
DEFAULT_SIGNUP_CREDITS_MICRO = 8_000_000
# Welcome campaigns this one supersedes. An account holding any of them has had
# its welcome grant and does not get v2 on top -- without this the boot
# backfill would credit every existing account another 8 Credits.
SUPERSEDED_SIGNUP_CREDIT_CAMPAIGNS = ("default_signup_credits_v1",)
DEFAULT_SIGNUP_CREDIT_SOURCE = "system_promotion"
DEFAULT_SIGNUP_CREDIT_REASON = "Automatic welcome Credits."

Expand Down Expand Up @@ -159,6 +169,10 @@ def get_balance(self, user_id: int) -> BalanceResult:
def grant_default_signup_credits(self, user_id: int) -> bool:
"""Give one account this campaign's welcome grant exactly once."""
user_id = int(user_id)
if self.store.has_promotion_grant(
user_id, SUPERSEDED_SIGNUP_CREDIT_CAMPAIGNS
):
return False
parts = {
"campaign_key": DEFAULT_SIGNUP_CREDIT_CAMPAIGN,
"user_id": user_id,
Expand Down
34 changes: 33 additions & 1 deletion dashboard/backend/infrastructure/llm/execution/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class ExecutionErrorCategory(StrEnum):
BILLING_FAILED = "billing_failed"
PROVIDER_QUOTA_EXHAUSTED = "provider_quota_exhausted"
ACCOUNT_RESTRICTED = "account_restricted"
INSUFFICIENT_CREDITS = "insufficient_credits"
WORKER_FAILED = "worker_failed"


Expand All @@ -37,6 +38,15 @@ class ExecutionErrorCategory(StrEnum):
"Your Credits account is paused. Add Credits to settle model usage "
"or contact an administrator."
),
# The balance cannot cover the next call's reservation. Kept apart from
# BILLING_FAILED, which it used to fall into via the catch-all in
# ``_execute_platform``: "you are out of Credits" is the user's to fix,
# "billing broke" is ours, and one message for both sent users hunting
# for a bug that was an empty balance.
ExecutionErrorCategory.INSUFFICIENT_CREDITS: (
"Not enough ATL Credits to continue this run. Add Credits on the "
"Credits page, choose a lower-cost model, or use your own API key."
),
ExecutionErrorCategory.WORKER_FAILED: "The model worker failed before completion.",
}

Expand Down Expand Up @@ -72,6 +82,10 @@ def __init__(
):
allowed_message = message
self.safe_message = allowed_message or _SAFE_MESSAGES[self.category]
# Set by ``account_restricted``; lets a process boundary that can carry
# only fixed tokens (the backtest child's run_failed line) rebuild the
# restriction-specific message rather than the generic one.
self.restriction_reason: str | None = None
super().__init__(self.safe_message)

@classmethod
Expand All @@ -93,7 +107,10 @@ def account_restricted(
message = _SAFE_RESTRICTED_MESSAGES.get(
reason, _SAFE_MESSAGES[ExecutionErrorCategory.ACCOUNT_RESTRICTED]
)
return cls(ExecutionErrorCategory.ACCOUNT_RESTRICTED, message)
error = cls(ExecutionErrorCategory.ACCOUNT_RESTRICTED, message)
if reason in _SAFE_RESTRICTED_MESSAGES:
error.restriction_reason = reason
return error


# A hosted (``fail_closed``) run aborts on any model error, because a billing or
Expand All @@ -117,9 +134,24 @@ def is_transient_provider_failure(error: BaseException) -> bool:
)


def run_failed_line(error: LLMExecutionError) -> str:
"""The line a backtest child prints when a model call ends its run.

Parsed by ``api/routers/backtests._child_llm_failure``. Fixed tokens only
(enum values), never upstream text, and the ``ERROR: llm.`` prefix makes
the parent relay it to the service log live.
"""
line = f"ERROR: llm.run_failed category={error.category.value}"
reason = getattr(error, "restriction_reason", None)
if reason:
line += f" reason={reason}"
return line


__all__ = [
"ExecutionErrorCategory",
"LLMExecutionError",
"RetryHint",
"is_transient_provider_failure",
"run_failed_line",
]
83 changes: 67 additions & 16 deletions dashboard/backend/infrastructure/llm/execution/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from dashboard.backend.domain.credits.service import CreditsService
from dashboard.backend.domain.credits.repository_common import (
CreditAccountRestrictedStoreError,
InsufficientCreditsError,
)
from dashboard.backend.domain.analytics import instrumentation as analytics_instrumentation
from dashboard.backend.domain.analytics.usage_cost import model_usage_properties
Expand Down Expand Up @@ -149,6 +150,11 @@ def _pricing_snapshot_for(model_id: str, provider_id: str) -> PricingSnapshot:
return PricingSnapshot.from_model(model_id, provider_id)


def prompt_token_ceiling_for_bytes(prompt_bytes: int) -> int:
"""The input-token ceiling a reservation charges for ``prompt_bytes``."""
return max(0, int(prompt_bytes)) + _PROMPT_FRAMING_TOKEN_ALLOWANCE


def _prompt_token_ceiling(request: LLMExecutionRequest) -> int:
payload = {
"system_message": request.system_message,
Expand All @@ -159,7 +165,38 @@ def _prompt_token_ceiling(request: LLMExecutionRequest) -> int:
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
return len(encoded) + _PROMPT_FRAMING_TOKEN_ALLOWANCE
return prompt_token_ceiling_for_bytes(len(encoded))


def _ceiling_micro(
input_tokens: int, output_tokens: int, pricing_snapshot: PricingSnapshot
) -> int | None:
ceiling_cost = estimate_cost_from_snapshot(
pricing_snapshot,
LLMUsage(input_tokens=int(input_tokens), output_tokens=int(output_tokens)),
)
if ceiling_cost is None:
return None
return credits_micro_for_usd(ceiling_cost)


def reservation_ceiling_micro_for(
*,
prompt_bytes: int,
max_output_tokens: int,
pricing_snapshot: PricingSnapshot,
) -> int | None:
"""The reservation one request of this shape holds, or None if unpriceable.

Shared with the backtest launch preflight so the two cannot drift: its
"the last call must find a whole ceiling free" is only true while it prices
exactly what ``_reservation_ceiling_micro`` does.
"""
return _ceiling_micro(
prompt_token_ceiling_for_bytes(prompt_bytes),
max_output_tokens,
pricing_snapshot,
)


def _reservation_ceiling_micro(
Expand All @@ -168,14 +205,32 @@ def _reservation_ceiling_micro(
) -> int:
"""Return the maximum billable cost for this one provider request."""

ceiling_usage = LLMUsage(
input_tokens=_prompt_token_ceiling(request),
output_tokens=request.usage_policy.max_output_tokens,
ceiling = _ceiling_micro(
_prompt_token_ceiling(request),
request.usage_policy.max_output_tokens,
pricing_snapshot,
)
ceiling_cost = estimate_cost_from_snapshot(pricing_snapshot, ceiling_usage)
if ceiling_cost is None:
if ceiling is None:
raise LLMExecutionError(ExecutionErrorCategory.BILLING_FAILED)
return credits_micro_for_usd(ceiling_cost)
return ceiling


def analytics_error_category(category: ExecutionErrorCategory) -> str:
"""The analytics ``error_category`` for an execution failure category.

Module-level so the backtest parent can label a child's failure without
constructing (or depending on) the service.
"""
return {
ExecutionErrorCategory.CREDENTIAL_MISSING: "credential_missing",
ExecutionErrorCategory.CREDENTIAL_INVALID: "credential_invalid",
ExecutionErrorCategory.PROVIDER_UNAVAILABLE: "provider_unavailable",
ExecutionErrorCategory.PROVIDER_TIMEOUT: "provider_timeout",
ExecutionErrorCategory.PROVIDER_QUOTA_EXHAUSTED: "provider_quota_exhausted",
ExecutionErrorCategory.BILLING_FAILED: "credits_unavailable",
ExecutionErrorCategory.ACCOUNT_RESTRICTED: "account_restricted",
ExecutionErrorCategory.INSUFFICIENT_CREDITS: "credits_insufficient",
}.get(category, "internal_error")


class LLMExecutionService:
Expand Down Expand Up @@ -244,15 +299,7 @@ def execute(self, request: LLMExecutionRequest) -> LLMExecutionResult:
def _analytics_error_category(
category: ExecutionErrorCategory,
) -> str:
return {
ExecutionErrorCategory.CREDENTIAL_MISSING: "credential_missing",
ExecutionErrorCategory.CREDENTIAL_INVALID: "credential_invalid",
ExecutionErrorCategory.PROVIDER_UNAVAILABLE: "provider_unavailable",
ExecutionErrorCategory.PROVIDER_TIMEOUT: "provider_timeout",
ExecutionErrorCategory.PROVIDER_QUOTA_EXHAUSTED: "provider_quota_exhausted",
ExecutionErrorCategory.BILLING_FAILED: "credits_unavailable",
ExecutionErrorCategory.ACCOUNT_RESTRICTED: "account_restricted",
}.get(category, "internal_error")
return analytics_error_category(category)

@staticmethod
def _emit_model_usage(
Expand Down Expand Up @@ -345,6 +392,10 @@ def _execute_platform(
raise LLMExecutionError.account_restricted(
reason, outstanding_micro
) from exc
except InsufficientCreditsError as exc:
raise LLMExecutionError(
ExecutionErrorCategory.INSUFFICIENT_CREDITS
) from exc
reservation_id = reservation.reservation_id
self._platform_runs.add(request.run_id)
if reservation.status != "open":
Expand Down
16 changes: 16 additions & 0 deletions dashboard/backend/tests/domain/analytics/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,3 +300,19 @@ def test_safe_error_accepts_provider_quota_exhausted():
)
)
assert record.error_category == "provider_quota_exhausted"


def test_every_execution_failure_category_maps_into_the_allow_list():
# `account_restricted` was mapped but not admitted, so emit_safe_error_event
# raised inside its own guard and every restricted-account failure was
# dropped with a WARNING line. Any category added to the mapping must be
# admitted here in the same change.
from dashboard.backend.infrastructure.llm.execution.errors import (
ExecutionErrorCategory,
)
from dashboard.backend.infrastructure.llm.execution.service import (
analytics_error_category,
)

for category in ExecutionErrorCategory:
assert analytics_error_category(category) in ALLOWED_ERROR_CATEGORIES, category
20 changes: 20 additions & 0 deletions dashboard/backend/tests/domain/analytics/test_states.py
Original file line number Diff line number Diff line change
Expand Up @@ -649,3 +649,23 @@ def test_value_snapshot_repair_isolates_one_user_failure(tmp_path, capsys):
assert "analytics.value_snapshot_repair_failed" in output
assert "category=RuntimeError" in output
assert "private first-user failure" not in output


def test_a_restricted_account_is_blocked_like_an_empty_balance(tmp_path):
service, store = _fixture(tmp_path)
_run(service, "backtest_failed", 0, NOW - timedelta(hours=2), error_category="internal_error")
blocked = service.record_server_event(
event_name="safe_error_recorded",
user_id=1,
source_event_id="resource:safe_error_recorded:run-restricted:account_restricted",
source_record_type="run",
source_record_id="run-restricted",
error_category="account_restricted",
occurred_at=NOW - timedelta(hours=1),
).event

snapshot = calculate_user_state(1, now=NOW, store=store)

assert snapshot.status == "blocked"
assert snapshot.reason_code == "billing_lane_unavailable"
assert snapshot.evidence_event_ids == [blocked.event_id]
Loading
Loading