From 30a5c13f06b368cd2c89b1a8e7a2d02a16288acd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 15:18:34 +0900 Subject: [PATCH 1/4] test(automation): reproduce mention sweep rate-limit amplification --- tests/test_agent_mention_rate_limit.py | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_agent_mention_rate_limit.py diff --git a/tests/test_agent_mention_rate_limit.py b/tests/test_agent_mention_rate_limit.py new file mode 100644 index 000000000..67b30bbe5 --- /dev/null +++ b/tests/test_agent_mention_rate_limit.py @@ -0,0 +1,72 @@ +"""Fail-fast regressions for exhausted GitHub mention-router API budgets.""" + +from __future__ import annotations + +import importlib +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" / "ci" +sys.path.insert(0, str(SCRIPTS)) + + +def module(): + """Reload the sweep module for isolated monkeypatching.""" + + return importlib.reload(importlib.import_module("agent_mention_sweep")) + + +def test_primary_rate_limit_exhaustion_stops_the_sweep(monkeypatch) -> None: + """A shared installation budget exhaustion must stop further API work.""" + + sweep = module() + issues = [ + {"repository": "ContextualWisdomLab/first", "number": 1}, + {"repository": "ContextualWisdomLab/second", "number": 2}, + ] + monkeypatch.setattr( + sweep, + "list_recent_pull_requests", + lambda *args, **kwargs: iter(issues), + ) + visited: list[str] = [] + + def build_requests(client, *, issue, since): + del client, since + visited.append(issue["repository"]) + raise RuntimeError( + "gh: API rate limit exceeded for installation ID 141441800 (HTTP 403)" + ) + + monkeypatch.setattr(sweep, "build_requests_for_pull_request", build_requests) + metrics = sweep.SweepMetrics() + + with pytest.raises(sweep.SweepRateLimitExhausted, match="rate limit"): + sweep.sweep( + target_client=object(), + dispatch_client=object(), + organization="ContextualWisdomLab", + repository_source="installation", + lookback_hours=24, + max_dispatches=5, + opencode_allowlist=frozenset(), + now=datetime(2026, 8, 15, tzinfo=timezone.utc), + metrics=metrics, + ) + + assert visited == ["ContextualWisdomLab/first"] + assert metrics.failures == 1 + + +def test_secondary_rate_limit_exhaustion_is_global() -> None: + """Secondary-limit messages are classified as sweep-global exhaustion.""" + + sweep = module() + assert sweep.is_rate_limit_exhaustion( + RuntimeError("You have exceeded a secondary rate limit. Please retry later.") + ) + assert not sweep.is_rate_limit_exhaustion(RuntimeError("Resource not accessible")) From dd9a8d294006e9c5da921804444bc1dcbd2079e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 15:19:12 +0900 Subject: [PATCH 2/4] fix(automation): stop mention sweep after shared rate-limit exhaustion --- scripts/ci/agent_mention_sweep.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/ci/agent_mention_sweep.py b/scripts/ci/agent_mention_sweep.py index 9b64909a0..891017e02 100644 --- a/scripts/ci/agent_mention_sweep.py +++ b/scripts/ci/agent_mention_sweep.py @@ -30,6 +30,20 @@ class SweepMetrics: failures: int = 0 +class SweepRateLimitExhausted(RuntimeError): + """Signal that shared GitHub API capacity is unavailable for this sweep.""" + + +def is_rate_limit_exhaustion(error: Exception) -> bool: + """Return whether an API error says the shared primary/secondary budget is exhausted.""" + + message = " ".join(str(error).split()).casefold() + return ( + "api rate limit exceeded" in message + or "secondary rate limit" in message + ) + + def parse_timestamp(value: str) -> datetime: """Parse one GitHub ISO-8601 timestamp into timezone-aware UTC.""" @@ -301,13 +315,18 @@ def sweep( dispatched = 0 def record_failure(scope: str, error: Exception) -> None: - """Record one isolated error and preserve the remaining sweep.""" + """Record isolated errors but stop when the shared API budget is exhausted.""" counters.failures += 1 message = " ".join(str(error).split()) or error.__class__.__name__ print( f"::warning::Agent mention sweep skipped {scope}: {message[:1000]}" ) + if is_rate_limit_exhaustion(error): + raise SweepRateLimitExhausted( + "GitHub API rate limit exhausted; stopping organization sweep " + "to preserve the shared installation budget" + ) from error for issue in list_recent_pull_requests( target_client, From 965896e2748e8493b150044b34baf5117cf95db4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 15:19:38 +0900 Subject: [PATCH 3/4] docs(automation): record mention sweep rate-limit boundary --- .../agent-mention-rate-limit-fail-fast.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/doctoring/agent-mention-rate-limit-fail-fast.md diff --git a/docs/doctoring/agent-mention-rate-limit-fail-fast.md b/docs/doctoring/agent-mention-rate-limit-fail-fast.md new file mode 100644 index 000000000..9d7e4e3d5 --- /dev/null +++ b/docs/doctoring/agent-mention-rate-limit-fail-fast.md @@ -0,0 +1,35 @@ +# Agent mention sweep rate-limit fail-fast boundary + +Updated: 2026-08-15 + +## Incident + +Scheduled `Review Agent Mention Router` run `31868885733` exhausted the OpenCode GitHub App installation REST budget before processing the requested review queue. The sweep continued traversing repositories after the first installation-wide `API rate limit exceeded` response and finished with zero dispatches plus 116 isolated failures. Repeating requests after the shared budget is exhausted cannot recover candidate-local work and consumes runner time while obscuring the single control-plane cause. + +## Decision + +Treat explicit GitHub primary- or secondary-rate-limit messages as **sweep-global capacity exhaustion**, not candidate-local failures. The sweep records the first failed scope, then raises `SweepRateLimitExhausted` immediately. Ordinary repository, pull-request, review, acknowledgement, and dispatch failures remain isolated exactly as before. + +This change is intentionally narrow. It does not retry, sleep, change credentials, widen permissions, alter the canonical invocation key, modify the exact-name artifact ledger, or claim that a failed request was dispatched. A later scheduled invocation may run after GitHub restores capacity. Interactive/local routing and the separate concurrency-isolation repair remain independent control-plane lanes. + +## Why fail-fast + +GitHub documents that installation access tokens share an installation-level primary REST budget. When a primary limit is exceeded, requests return HTTP 403 or 429 and callers should not retry until the reset time. GitHub also states that integrations should stop and wait on secondary-rate-limit responses; continuing to make requests while rate-limited may lead to integration bans. The current sweep cannot safely infer reset headers from the `gh` exception string, so the bounded action is to stop the current scheduled traversal rather than amplify the exhausted state. + +## Verification contract + +- a synthetic installation-wide primary-limit error on the first PR aborts before the second PR is touched; +- exactly one failure is recorded for the first exhausted scope; +- secondary-rate-limit messages are classified as sweep-global exhaustion; +- unrelated authorization/resource errors remain candidate-local and preserve existing failure isolation; +- the permanent agent-mention quality suite continues to require 100% owned production statement/branch and public docstring coverage. + +## Rollback + +Revert `SweepRateLimitExhausted`, `is_rate_limit_exhaustion`, and their focused regression if GitHub changes the CLI error contract or the router gains structured response-header handling. Do not restore repeated API calls after a proven shared rate-limit exhaustion without an equivalent bounded backoff/stop mechanism. + +## References + +GitHub. (n.d.). *Rate limits for the REST API*. GitHub Docs. Retrieved August 15, 2026, from https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api + +GitHub. (n.d.). *Rate limits for GitHub Apps*. GitHub Docs. Retrieved August 15, 2026, from https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/rate-limits-for-github-apps From 0cccd7f92ca5c59306a17afffc8dd11ddb7508f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 16:40:18 +0900 Subject: [PATCH 4/4] fix(mentions): stop cleanly on exhausted API budget --- .../agent-mention-rate-limit-fail-fast.md | 3 ++ scripts/ci/agent_mention_sweep.py | 34 +++++++++++-------- tests/test_agent_mention_rate_limit.py | 21 ++++++++++++ 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/docs/doctoring/agent-mention-rate-limit-fail-fast.md b/docs/doctoring/agent-mention-rate-limit-fail-fast.md index 9d7e4e3d5..954127005 100644 --- a/docs/doctoring/agent-mention-rate-limit-fail-fast.md +++ b/docs/doctoring/agent-mention-rate-limit-fail-fast.md @@ -12,6 +12,9 @@ Treat explicit GitHub primary- or secondary-rate-limit messages as **sweep-globa This change is intentionally narrow. It does not retry, sleep, change credentials, widen permissions, alter the canonical invocation key, modify the exact-name artifact ledger, or claim that a failed request was dispatched. A later scheduled invocation may run after GitHub restores capacity. Interactive/local routing and the separate concurrency-isolation repair remain independent control-plane lanes. +The scheduled CLI emits an error with the budget-reset action and exits 1, so +operators see a bounded failure rather than an unhandled traceback. + ## Why fail-fast GitHub documents that installation access tokens share an installation-level primary REST budget. When a primary limit is exceeded, requests return HTTP 403 or 429 and callers should not retry until the reset time. GitHub also states that integrations should stop and wait on secondary-rate-limit responses; continuing to make requests while rate-limited may lead to integration bans. The current sweep cannot safely infer reset headers from the `gh` exception string, so the bounded action is to stop the current scheduled traversal rather than amplify the exhausted state. diff --git a/scripts/ci/agent_mention_sweep.py b/scripts/ci/agent_mention_sweep.py index 891017e02..3bd7bee43 100644 --- a/scripts/ci/agent_mention_sweep.py +++ b/scripts/ci/agent_mention_sweep.py @@ -40,6 +40,7 @@ def is_rate_limit_exhaustion(error: Exception) -> bool: message = " ".join(str(error).split()).casefold() return ( "api rate limit exceeded" in message + or "api rate limit already exceeded" in message or "secondary rate limit" in message ) @@ -325,7 +326,8 @@ def record_failure(scope: str, error: Exception) -> None: if is_rate_limit_exhaustion(error): raise SweepRateLimitExhausted( "GitHub API rate limit exhausted; stopping organization sweep " - "to preserve the shared installation budget" + "to preserve the shared installation budget; wait for the " + "budget reset before retrying" ) from error for issue in list_recent_pull_requests( @@ -393,19 +395,23 @@ def main(argv: Sequence[str] | None = None) -> int: os.environ.get("OPENCODE_REPOSITORY_DISPATCH_TARGETS", "") ) metrics = SweepMetrics() - sweep( - target_client=GitHubClient( - os.environ.get("TARGET_REPOSITORY_TOKEN", "") - ), - dispatch_client=GitHubClient(os.environ.get("AGENT_DISPATCH_TOKEN", "")), - organization=args.organization, - repository_source=args.repository_source, - lookback_hours=args.lookback_hours, - max_dispatches=args.max_dispatches, - opencode_allowlist=allowlist, - dry_run=args.dry_run, - metrics=metrics, - ) + try: + sweep( + target_client=GitHubClient( + os.environ.get("TARGET_REPOSITORY_TOKEN", "") + ), + dispatch_client=GitHubClient(os.environ.get("AGENT_DISPATCH_TOKEN", "")), + organization=args.organization, + repository_source=args.repository_source, + lookback_hours=args.lookback_hours, + max_dispatches=args.max_dispatches, + opencode_allowlist=allowlist, + dry_run=args.dry_run, + metrics=metrics, + ) + except SweepRateLimitExhausted as exc: + print(f"::error::{exc}") + return 1 return 1 if metrics.failures else 0 diff --git a/tests/test_agent_mention_rate_limit.py b/tests/test_agent_mention_rate_limit.py index 67b30bbe5..46455282a 100644 --- a/tests/test_agent_mention_rate_limit.py +++ b/tests/test_agent_mention_rate_limit.py @@ -69,4 +69,25 @@ def test_secondary_rate_limit_exhaustion_is_global() -> None: assert sweep.is_rate_limit_exhaustion( RuntimeError("You have exceeded a secondary rate limit. Please retry later.") ) + assert sweep.is_rate_limit_exhaustion( + RuntimeError("API rate limit already exceeded for installation ID 141441800") + ) assert not sweep.is_rate_limit_exhaustion(RuntimeError("Resource not accessible")) + + +def test_main_reports_rate_limit_reset_and_returns_failure(monkeypatch, capsys) -> None: + """The scheduled CLI reports the shared-budget stop without a traceback.""" + + sweep = module() + monkeypatch.setenv("TARGET_REPOSITORY_TOKEN", "target") + monkeypatch.setenv("AGENT_DISPATCH_TOKEN", "dispatch") + def fail_for_rate_limit(**kwargs): + del kwargs + raise sweep.SweepRateLimitExhausted( + "wait for the budget reset before retrying" + ) + + monkeypatch.setattr(sweep, "sweep", fail_for_rate_limit) + + assert sweep.main([]) == 1 + assert "wait for the budget reset before retrying" in capsys.readouterr().out