fix(sdk): do not retry a 429 quota_exceeded of the evals routes

docs/sdk-conformance.md

@@ -322,9 +322,12 @@ timeout, a network error, a 429, or a 5xx. The server skips an item run that it
 makes the experiment tries again only after a 429, which the rate limit answers before any work: the
 server answers 503 when its database work times out, which can be after the commit. A retry waits
 for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1
-second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript
-request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a
-score. The Python timeouts apply to each socket operation of urllib.
+second and then 2 seconds. A 429 with the code `quota_exceeded`, or with a `Retry-After` of more
+than 60 seconds, gets no retry: the request fails at once with the error of the server. A quota
+resets at the start of the next month, or never for a gift budget. `POST /v1/scores` makes one
+request, with no retry. The TypeScript request timeouts abort the request: 30 seconds for the
+experiment requests and 10 seconds for a score. The Python timeouts apply to each socket operation
+of urllib.
 
 Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1).
 `concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names

packages/sdk/README.md

@@ -306,9 +306,12 @@ const result = await runExperiment({
   `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the
   completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server
   skips an item run that it has. The request that makes the experiment goes again only after a
-  429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it
-  made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then
-  ends the experiment with the item runs that it got, or fails it when it got none.
+  429, thus a lost response or a 503 cannot make a second experiment. No request goes again after
+  a 429 `quota_exceeded`, which a cap of the plan gives, or after a 429 whose `Retry-After` is
+  more than 60 seconds: runExperiment rejects at once with the message of the server. When the run
+  stops after it made the experiment, runExperiment sends the completion before it rejects:
+  telemetry.dev then ends the experiment with the item runs that it got, or fails it when it got
+  none.
 - The key needs Run experiments in its Access on the API keys page (the `datasets:read` and
   `experiments:write` scopes). runExperiment uses the same
   API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch`

packages/sdk/src/api.ts

@@ -21,7 +21,7 @@ export interface Api {
 }
 
 /** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */
-export type ApiErrorType<E extends Error> = new (
+export type ApiErrorType<E extends ApiError> = new (
   code: string,
   message: string,
   status?: number,
@@ -42,11 +42,16 @@ interface RequestOptions {
   attempts?: number;
 }
 
-type Attempt =
-  | { ok: true; body: unknown }
-  | { ok: false; error: Error; status?: number; retryAfterMs?: number };
+type ApiError = Error & { readonly code: string };
 
-/** A longer Retry-After waits only this long: the API counts requests by the minute. */
+type Failure = { ok: false; error: ApiError; status?: number; retryAfterMs?: number };
+
+type Attempt = { ok: true; body: unknown } | Failure;
+
+/**
+ * The longest wait before a retry: the API counts requests by the minute. A 429 with a longer
+ * Retry-After fails at once, and a 503 with a longer one waits only this long.
+ */
 const MAX_RETRY_AFTER_MS = 60_000;
 /** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */
 const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/;
@@ -75,9 +80,10 @@ export function resolveApi(options: ApiOptions): Api {
  * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response
  * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_<status>`.
  * A failure that `options.idempotent` allows tries again, after the Retry-After of the response
- * or after 1 and then 2 seconds.
+ * or after 1 and then 2 seconds. A 429 `quota_exceeded`, or a 429 with a Retry-After past 60
+ * seconds, fails at once.
  */
-export async function request<E extends Error>(
+export async function request<E extends ApiError>(
   api: Api & { apiKey: string },
   ErrorType: ApiErrorType<E>,
   method: "GET" | "POST",
@@ -99,7 +105,7 @@ export async function request<E extends Error>(
 
     if (result.ok) return result.body;
 
-    if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error;
+    if (attempt >= attempts || !retryable(result, options.idempotent)) throw result.error;
 
     await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS));
   }
@@ -130,7 +136,7 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
-async function send<E extends Error>(
+async function send<E extends ApiError>(
   api: Api & { apiKey: string },
   ErrorType: ApiErrorType<E>,
   method: "GET" | "POST",
@@ -196,8 +202,11 @@ async function send<E extends Error>(
   }
 }
 
-function retryable(status: number | undefined, idempotent: boolean): boolean {
-  if (status === 429) return true;
+function retryable({ error, status, retryAfterMs }: Failure, idempotent: boolean): boolean {
+  if (status === 429) {
+    // A monthly quota resets at the start of next month, and a gift budget never resets.
+    return error.code !== "quota_exceeded" && (retryAfterMs ?? 0) <= MAX_RETRY_AFTER_MS;
+  }
 
   return idempotent && (status === undefined || status >= 500);
 }

packages/sdk/tests/experiments.test.ts

@@ -741,6 +741,49 @@ test("a create that gets a 429 goes again after its Retry-After", async () => {
   expect(Date.now() - started).toBeGreaterThanOrEqual(2_000);
 });
 
+test.each([
+  [
+    "a create that gets a 429 quota_exceeded until next month",
+    "POST /v1/experiments",
+    "quota_exceeded",
+    "Monthly ingestion quota exceeded; upgrade your plan to continue.",
+    { "retry-after": "1468800" },
+  ],
+  [
+    "an item post that gets a 429 quota_exceeded with no Retry-After",
+    `POST /v1/experiments/${EXPERIMENT_ID}/items`,
+    "quota_exceeded",
+    "Lifetime gift ingestion budget exhausted; subscribe to continue. This budget does not reset monthly.",
+    {},
+  ],
+  [
+    "a create that gets a 429 with a Retry-After of 61 seconds",
+    "POST /v1/experiments",
+    "rate_limited",
+    "Too many requests for this project",
+    { "retry-after": "61" },
+  ],
+])("%s fails at once with the message of the server", async (_, route, code, message, headers) => {
+  vi.useFakeTimers();
+
+  try {
+    const api = fakeApi({
+      override: (request) =>
+        `${request.method} ${request.url.pathname}` === route
+          ? Response.json({ error: { code, message } }, { status: 429, headers })
+          : undefined,
+    });
+
+    const outcome = experiment(api).catch((caught: unknown) => caught);
+    await vi.advanceTimersByTimeAsync(5 * 60_000);
+
+    expect(await outcome).toMatchObject({ code, message, status: 429 });
+    expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(1);
+  } finally {
+    vi.useRealTimers();
+  }
+});
+
 test("waits until the server evaluators end the experiment", async () => {
   const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] });
 

sdks/python/README.md

@@ -387,10 +387,12 @@ result = telemetry_dev.run_experiment(
   `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the
   completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server
   skips an item run that it has. The request that makes the experiment goes again only after a
-  429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it
-  made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion
-  before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails
-  it when it got none.
+  429, thus a lost response or a 503 cannot make a second experiment. No request goes again after
+  a 429 `quota_exceeded`, which a cap of the plan gives, or after a 429 whose `Retry-After` is
+  more than 60 seconds: `run_experiment` raises at once with the message of the server. When the
+  run stops after it made the experiment, also after Ctrl-C or a cancellation, `run_experiment`
+  sends the completion before it raises: telemetry.dev then ends the experiment with the item runs
+  that it got, or fails it when it got none.
 - The key needs Run experiments in its Access on the API keys page (the `datasets:read` and
   `experiments:write` scopes). `run_experiment` uses the same
   API key and base URL as `get_prompt`, or its own `api_key` and `base_url`.

sdks/python/src/telemetry_dev/_api.py

@@ -20,7 +20,8 @@
 from ._config import SDK_VERSION, resolve_config
 from ._serialize import json_dumps
 
-# A longer Retry-After waits only this long: the API counts requests by the minute.
+# The longest wait before a retry: the API counts requests by the minute. A 429 with a longer
+# Retry-After fails at once, and a 503 with a longer one waits only this long.
 _MAX_RETRY_AFTER_S = 60.0
 # IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT.
 _HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII)
@@ -52,6 +53,7 @@ def resolve_api(api_key: str | None, base_url: str | None) -> Api:
 @dataclass(frozen=True)
 class _Failure:
     error: Exception
+    code: str | None
     status: int | None
     retry_after: float | None
 
@@ -74,7 +76,8 @@ def request(
     ``http_<status>``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a
     network error, or a 5xx tries again. Other requests try again only after a 429, which the rate
     limit answers before the server does any work: a 503 can come after the work committed. A
-    retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout
+    retry waits for the Retry-After of the response, or for 1 and then 2 seconds. A 429
+    ``quota_exceeded``, or a 429 with a Retry-After past 60 seconds, raises at once. The timeout
     applies to each socket operation of urllib.
     """
     url = f"{api.base_url}{path}"
@@ -92,7 +95,7 @@ def request(
         result = _send(error_type, method, url, path, data, headers, timeout_s)
         if not isinstance(result, _Failure):
             return result
-        if attempt >= attempts or not _retryable(result.status, idempotent):
+        if attempt >= attempts or not _retryable(result, idempotent):
             raise result.error
         time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S))
         attempt += 1
@@ -128,18 +131,7 @@ def redirect_request(
 
 def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E:
     """The error of a response that is not 2xx: its API code and message, or ``http_<status>``."""
-    try:
-        error = json.loads(body).get("error")
-    except (ValueError, AttributeError):
-        error = None
-    fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {}
-    code = fields.get("code")
-    message = fields.get("message")
-    return error_type(
-        code if isinstance(code, str) else f"http_{status}",
-        message if isinstance(message, str) else f"telemetry.dev returned {status}",
-        status,
-    )
+    return error_type(*_code_and_message(status, body), status)
 
 
 def retry_after(status: int, value: str | None) -> float | None:
@@ -175,8 +167,10 @@ def _send(
             body: bytes = response.read()
     except urllib.error.HTTPError as exc:
         with exc:
+            code, message = _code_and_message(exc.code, exc.read())
             return _Failure(
-                api_error(error_type, exc.code, exc.read()),
+                error_type(code, message, exc.code),
+                code,
                 exc.code,
                 retry_after(exc.code, exc.headers.get("Retry-After")),
             )
@@ -190,14 +184,29 @@ def _send(
             else error_type("network_error", "Could not reach telemetry.dev", None)
         )
         error.__cause__ = exc
-        return _Failure(error, None, None)
+        return _Failure(error, None, None, None)
     try:
         return json.loads(body)
     except ValueError:
         return None
 
 
-def _retryable(status: int | None, idempotent: bool) -> bool:
-    if status == 429:
-        return True
-    return idempotent and (status is None or status >= 500)
+def _code_and_message(status: int, body: bytes) -> tuple[str, str]:
+    try:
+        error = json.loads(body).get("error")
+    except (ValueError, AttributeError):
+        error = None
+    fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {}
+    code = fields.get("code")
+    message = fields.get("message")
+    return (
+        code if isinstance(code, str) else f"http_{status}",
+        message if isinstance(message, str) else f"telemetry.dev returned {status}",
+    )
+
+
+def _retryable(failure: _Failure, idempotent: bool) -> bool:
+    if failure.status == 429:
+        # A monthly quota resets at the start of next month, and a gift budget never resets.
+        return failure.code != "quota_exceeded" and (failure.retry_after or 0) <= _MAX_RETRY_AFTER_S
+    return idempotent and (failure.status is None or failure.status >= 500)

sdks/python/tests/test_experiments.py

@@ -11,6 +11,7 @@
 import pytest
 
 import telemetry_dev
+import telemetry_dev._api as api_module
 import telemetry_dev._experiments as experiments_module
 from telemetry_dev import (
     ExperimentError,
@@ -641,6 +642,55 @@ def limit(request: ApiRequest) -> Reply | None:
     assert time.monotonic() - started >= 2
 
 
+@pytest.mark.parametrize(
+    ("route", "code", "message", "headers"),
+    [
+        (
+            "POST /v1/experiments",
+            "quota_exceeded",
+            "Monthly ingestion quota exceeded; upgrade your plan to continue.",
+            {"retry-after": "1468800"},
+        ),
+        (
+            f"POST /v1/experiments/{EXPERIMENT_ID}/items",
+            "quota_exceeded",
+            "Lifetime gift ingestion budget exhausted; subscribe to continue. This budget does "
+            "not reset monthly.",
+            {},
+        ),
+        (
+            "POST /v1/experiments",
+            "rate_limited",
+            "Too many requests for this project",
+            {"retry-after": "61"},
+        ),
+    ],
+    ids=[
+        "a create that gets a 429 quota_exceeded until next month",
+        "an item post that gets a 429 quota_exceeded with no retry-after",
+        "a create that gets a 429 with a retry-after of 61 seconds",
+    ],
+)
+def test_a_429_that_a_retry_cannot_pass_raises_at_once_with_the_message_of_the_server(
+    monkeypatch: pytest.MonkeyPatch, route: str, code: str, message: str, headers: dict[str, str]
+) -> None:
+    clock = FakeClock()
+    monkeypatch.setattr(api_module, "time", clock)
+
+    def refuse(request: ApiRequest) -> Reply | None:
+        if request.route != route:
+            return None
+        return answer_with(429, {"error": {"code": code, "message": message}}, headers)
+
+    with fake_api(override=refuse) as api:
+        with pytest.raises(ExperimentError) as caught:
+            experiment(api)
+
+    assert (caught.value.code, caught.value.message, caught.value.status) == (code, message, 429)
+    assert api.routes().count(route) == 1
+    assert clock.now == 0
+
+
 def test_waits_until_the_server_evaluators_end_the_experiment() -> None:
     with fake_api(results=[result(status="running", itemsDone=1), result()]) as api:
         outcome = experiment(api)