diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 645ce55..3c49005 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -322,9 +322,12 @@ timeout, a network error, a 429, or a 5xx. The server skips an item run that it makes the experiment tries again only after a 429, which the rate limit answers before any work: the server answers 503 when its database work times out, which can be after the commit. A retry waits for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 -second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript -request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a -score. The Python timeouts apply to each socket operation of urllib. +second and then 2 seconds. A 429 with the code `quota_exceeded`, or with a `Retry-After` of more +than 60 seconds, gets no retry: the request fails at once with the error of the server. A quota +resets at the start of the next month, or never for a gift budget. `POST /v1/scores` makes one +request, with no retry. The TypeScript request timeouts abort the request: 30 seconds for the +experiment requests and 10 seconds for a score. The Python timeouts apply to each socket operation +of urllib. Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). `concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ecb6c4b..d06f4f2 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -306,9 +306,12 @@ const result = await runExperiment({ `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The request that makes the experiment goes again only after a - 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it - made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then - ends the experiment with the item runs that it got, or fails it when it got none. + 429, thus a lost response or a 503 cannot make a second experiment. No request goes again after + a 429 `quota_exceeded`, which a cap of the plan gives, or after a 429 whose `Retry-After` is + more than 60 seconds: runExperiment rejects at once with the message of the server. When the run + stops after it made the experiment, runExperiment sends the completion before it rejects: + telemetry.dev then ends the experiment with the item runs that it got, or fails it when it got + none. - The key needs Run experiments in its Access on the API keys page (the `datasets:read` and `experiments:write` scopes). runExperiment uses the same API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts index dd7fac6..4874f43 100644 --- a/packages/sdk/src/api.ts +++ b/packages/sdk/src/api.ts @@ -21,7 +21,7 @@ export interface Api { } /** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */ -export type ApiErrorType = new ( +export type ApiErrorType = new ( code: string, message: string, status?: number, @@ -42,11 +42,16 @@ interface RequestOptions { attempts?: number; } -type Attempt = - | { ok: true; body: unknown } - | { ok: false; error: Error; status?: number; retryAfterMs?: number }; +type ApiError = Error & { readonly code: string }; -/** A longer Retry-After waits only this long: the API counts requests by the minute. */ +type Failure = { ok: false; error: ApiError; status?: number; retryAfterMs?: number }; + +type Attempt = { ok: true; body: unknown } | Failure; + +/** + * The longest wait before a retry: the API counts requests by the minute. A 429 with a longer + * Retry-After fails at once, and a 503 with a longer one waits only this long. + */ const MAX_RETRY_AFTER_MS = 60_000; /** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; @@ -75,9 +80,10 @@ export function resolveApi(options: ApiOptions): Api { * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_`. * A failure that `options.idempotent` allows tries again, after the Retry-After of the response - * or after 1 and then 2 seconds. + * or after 1 and then 2 seconds. A 429 `quota_exceeded`, or a 429 with a Retry-After past 60 + * seconds, fails at once. */ -export async function request( +export async function request( api: Api & { apiKey: string }, ErrorType: ApiErrorType, method: "GET" | "POST", @@ -99,7 +105,7 @@ export async function request( if (result.ok) return result.body; - if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error; + if (attempt >= attempts || !retryable(result, options.idempotent)) throw result.error; await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS)); } @@ -130,7 +136,7 @@ export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -async function send( +async function send( api: Api & { apiKey: string }, ErrorType: ApiErrorType, method: "GET" | "POST", @@ -196,8 +202,11 @@ async function send( } } -function retryable(status: number | undefined, idempotent: boolean): boolean { - if (status === 429) return true; +function retryable({ error, status, retryAfterMs }: Failure, idempotent: boolean): boolean { + if (status === 429) { + // A monthly quota resets at the start of next month, and a gift budget never resets. + return error.code !== "quota_exceeded" && (retryAfterMs ?? 0) <= MAX_RETRY_AFTER_MS; + } return idempotent && (status === undefined || status >= 500); } diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts index b114b5f..ebcfdaf 100644 --- a/packages/sdk/tests/experiments.test.ts +++ b/packages/sdk/tests/experiments.test.ts @@ -741,6 +741,49 @@ test("a create that gets a 429 goes again after its Retry-After", async () => { expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); }); +test.each([ + [ + "a create that gets a 429 quota_exceeded until next month", + "POST /v1/experiments", + "quota_exceeded", + "Monthly ingestion quota exceeded; upgrade your plan to continue.", + { "retry-after": "1468800" }, + ], + [ + "an item post that gets a 429 quota_exceeded with no Retry-After", + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + "quota_exceeded", + "Lifetime gift ingestion budget exhausted; subscribe to continue. This budget does not reset monthly.", + {}, + ], + [ + "a create that gets a 429 with a Retry-After of 61 seconds", + "POST /v1/experiments", + "rate_limited", + "Too many requests for this project", + { "retry-after": "61" }, + ], +])("%s fails at once with the message of the server", async (_, route, code, message, headers) => { + vi.useFakeTimers(); + + try { + const api = fakeApi({ + override: (request) => + `${request.method} ${request.url.pathname}` === route + ? Response.json({ error: { code, message } }, { status: 429, headers }) + : undefined, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(5 * 60_000); + + expect(await outcome).toMatchObject({ code, message, status: 429 }); + expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(1); + } finally { + vi.useRealTimers(); + } +}); + test("waits until the server evaluators end the experiment", async () => { const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] }); diff --git a/sdks/python/README.md b/sdks/python/README.md index 72aadb2..fbc03d2 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -387,10 +387,12 @@ result = telemetry_dev.run_experiment( `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The request that makes the experiment goes again only after a - 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it - made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion - before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails - it when it got none. + 429, thus a lost response or a 503 cannot make a second experiment. No request goes again after + a 429 `quota_exceeded`, which a cap of the plan gives, or after a 429 whose `Retry-After` is + more than 60 seconds: `run_experiment` raises at once with the message of the server. When the + run stops after it made the experiment, also after Ctrl-C or a cancellation, `run_experiment` + sends the completion before it raises: telemetry.dev then ends the experiment with the item runs + that it got, or fails it when it got none. - The key needs Run experiments in its Access on the API keys page (the `datasets:read` and `experiments:write` scopes). `run_experiment` uses the same API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py index 5dc3065..14fce8b 100644 --- a/sdks/python/src/telemetry_dev/_api.py +++ b/sdks/python/src/telemetry_dev/_api.py @@ -20,7 +20,8 @@ from ._config import SDK_VERSION, resolve_config from ._serialize import json_dumps -# A longer Retry-After waits only this long: the API counts requests by the minute. +# The longest wait before a retry: the API counts requests by the minute. A 429 with a longer +# Retry-After fails at once, and a 503 with a longer one waits only this long. _MAX_RETRY_AFTER_S = 60.0 # IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. _HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) @@ -52,6 +53,7 @@ def resolve_api(api_key: str | None, base_url: str | None) -> Api: @dataclass(frozen=True) class _Failure: error: Exception + code: str | None status: int | None retry_after: float | None @@ -74,7 +76,8 @@ def request( ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a network error, or a 5xx tries again. Other requests try again only after a 429, which the rate limit answers before the server does any work: a 503 can come after the work committed. A - retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout + retry waits for the Retry-After of the response, or for 1 and then 2 seconds. A 429 + ``quota_exceeded``, or a 429 with a Retry-After past 60 seconds, raises at once. The timeout applies to each socket operation of urllib. """ url = f"{api.base_url}{path}" @@ -92,7 +95,7 @@ def request( result = _send(error_type, method, url, path, data, headers, timeout_s) if not isinstance(result, _Failure): return result - if attempt >= attempts or not _retryable(result.status, idempotent): + if attempt >= attempts or not _retryable(result, idempotent): raise result.error time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S)) attempt += 1 @@ -128,18 +131,7 @@ def redirect_request( def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E: """The error of a response that is not 2xx: its API code and message, or ``http_``.""" - try: - error = json.loads(body).get("error") - except (ValueError, AttributeError): - error = None - fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} - code = fields.get("code") - message = fields.get("message") - return error_type( - code if isinstance(code, str) else f"http_{status}", - message if isinstance(message, str) else f"telemetry.dev returned {status}", - status, - ) + return error_type(*_code_and_message(status, body), status) def retry_after(status: int, value: str | None) -> float | None: @@ -175,8 +167,10 @@ def _send( body: bytes = response.read() except urllib.error.HTTPError as exc: with exc: + code, message = _code_and_message(exc.code, exc.read()) return _Failure( - api_error(error_type, exc.code, exc.read()), + error_type(code, message, exc.code), + code, exc.code, retry_after(exc.code, exc.headers.get("Retry-After")), ) @@ -190,14 +184,29 @@ def _send( else error_type("network_error", "Could not reach telemetry.dev", None) ) error.__cause__ = exc - return _Failure(error, None, None) + return _Failure(error, None, None, None) try: return json.loads(body) except ValueError: return None -def _retryable(status: int | None, idempotent: bool) -> bool: - if status == 429: - return True - return idempotent and (status is None or status >= 500) +def _code_and_message(status: int, body: bytes) -> tuple[str, str]: + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} + code = fields.get("code") + message = fields.get("message") + return ( + code if isinstance(code, str) else f"http_{status}", + message if isinstance(message, str) else f"telemetry.dev returned {status}", + ) + + +def _retryable(failure: _Failure, idempotent: bool) -> bool: + if failure.status == 429: + # A monthly quota resets at the start of next month, and a gift budget never resets. + return failure.code != "quota_exceeded" and (failure.retry_after or 0) <= _MAX_RETRY_AFTER_S + return idempotent and (failure.status is None or failure.status >= 500) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py index a3850c6..d6a4168 100644 --- a/sdks/python/tests/test_experiments.py +++ b/sdks/python/tests/test_experiments.py @@ -11,6 +11,7 @@ import pytest import telemetry_dev +import telemetry_dev._api as api_module import telemetry_dev._experiments as experiments_module from telemetry_dev import ( ExperimentError, @@ -641,6 +642,55 @@ def limit(request: ApiRequest) -> Reply | None: assert time.monotonic() - started >= 2 +@pytest.mark.parametrize( + ("route", "code", "message", "headers"), + [ + ( + "POST /v1/experiments", + "quota_exceeded", + "Monthly ingestion quota exceeded; upgrade your plan to continue.", + {"retry-after": "1468800"}, + ), + ( + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + "quota_exceeded", + "Lifetime gift ingestion budget exhausted; subscribe to continue. This budget does " + "not reset monthly.", + {}, + ), + ( + "POST /v1/experiments", + "rate_limited", + "Too many requests for this project", + {"retry-after": "61"}, + ), + ], + ids=[ + "a create that gets a 429 quota_exceeded until next month", + "an item post that gets a 429 quota_exceeded with no retry-after", + "a create that gets a 429 with a retry-after of 61 seconds", + ], +) +def test_a_429_that_a_retry_cannot_pass_raises_at_once_with_the_message_of_the_server( + monkeypatch: pytest.MonkeyPatch, route: str, code: str, message: str, headers: dict[str, str] +) -> None: + clock = FakeClock() + monkeypatch.setattr(api_module, "time", clock) + + def refuse(request: ApiRequest) -> Reply | None: + if request.route != route: + return None + return answer_with(429, {"error": {"code": code, "message": message}}, headers) + + with fake_api(override=refuse) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, caught.value.message, caught.value.status) == (code, message, 429) + assert api.routes().count(route) == 1 + assert clock.now == 0 + + def test_waits_until_the_server_evaluators_end_the_experiment() -> None: with fake_api(results=[result(status="running", itemsDone=1), result()]) as api: outcome = experiment(api)