A Mercor 5xx is content not obtained: exit 5, or 6 after rows, never 4

CHANGELOG.md

@@ -11,6 +11,17 @@ it from a bill or from a diff.
 
 ## [Unreleased]
 
+> **Behaviour change: a Mercor 5xx is no longer exit 4.** A page the site
+> answered with a server error used to be counted as served, and a run whose
+> only page was a 503 said "empty result" (exit 4). Now the page is retried
+> within the existing budget and then reported as content not obtained. That
+> is exit 5 on the first page and exit 6 after earlier rows, with stop reason
+> `target_http_5xx` and the status in `pages_target_http_error`. This applies
+> in Playwright, Selenium, pyppeteer and `scraper_api_client.py`, where the
+> status is the Scraper API's `http_code`. A Scraper API answer of
+> `status: "error"` with no target status is exit 5 too. A 404 is still
+> exit 4.
+
 ### Fixed
 - **`scraper_api_client.py` handles a Mercor 404 the way the browser engines
   do.** The exit code was already 4 and stays 4. A dead job answered through

README.md

@@ -230,6 +230,15 @@ than as an API error. A non-200 from the Scraper API itself (its own 404
 means an unknown task id; 402, 408 and 422 are its other refusals) is `5`,
 because then nothing was obtained from Mercor at all.
 
+**A 5xx from Mercor is `5` (or `6` after earlier rows), never `4`.** A server
+error says nothing about what the page holds. The page is fetched again
+within the existing retry budget. If every attempt is a 5xx, the page is
+failed (stop reason `target_http_5xx`), and the sidecar records the status
+per page in `pages_target_http_error`. That status is the navigation's own in
+the browser engines and `http_code` through the Scraper API. With no rows,
+nothing is written, so earlier good output and its sidecar stay as they were;
+`--allow-empty` writes a `failed` sidecar instead.
+
 A run that finds nothing **writes nothing** — last night's good output is
 never replaced with `[]`. `--allow-empty` opts out.
 

page_flow.py

@@ -365,6 +365,41 @@ def account_for_pages(outcomes):
     return read, failed, unreadable
 
 
+
+# The stop reason for a page the TARGET answered with a server error.
+TARGET_SERVER_ERROR = "target_http_5xx"
+
+
+def target_server_error(state: str, status) -> Optional[int]:
+    """The status, when a page ended as a target-site server error.
+
+    `detect_page_state` calls a 5xx with no records `unknown`, and `unknown`
+    is retried. When the retries are spent the page is still not the site's
+    answer. A 503 says nothing about whether the job or the index holds
+    anything, so it is content NOT obtained (exit 5, or 6 after earlier
+    rows), never the empty result exit 4 claims. A 404 is not here: that is
+    the site saying the address does not exist.
+    """
+    if state == "unknown" and isinstance(status, int) and status >= 500:
+        return status
+    return None
+
+
+def failure_stop_reason(outcome) -> str:
+    """Why a failed page ended the run, in the same words for every engine."""
+    if outcome.load_failed:
+        return "page_load_timeout"
+    if getattr(outcome, "server_error", None) is not None:
+        return TARGET_SERVER_ERROR
+    return f"blocked_{outcome.blocked_by}"
+
+
+def target_http_errors(outcomes) -> dict:
+    """{page number: target status} for every page that ended as a 5xx."""
+    return {str(o.page_num): o.server_error
+            for o in sorted(outcomes, key=lambda o: o.page_num)
+            if getattr(o, "server_error", None) is not None}
+
 def reconcile_stop_reason(stop_reason: str,
                           enumeration_complete: Optional[bool],
                           unreadable_pages=()) -> str:

playwright_scraper.py

@@ -129,6 +129,9 @@ class PageOutcome:
     products: List = field(default_factory=list)
     blocked_by: Optional[str] = None
     load_failed: bool = False
+    # The target's 5xx status when the page ended as a server error after
+    # its retries (see page_flow.target_server_error). Not ok: no content.
+    server_error: Optional[int] = None
     # The page_flow state this page came back as ("content", "blocked",
     # "challenge", "empty", "unknown"). Carried so the caller can tell an
     # EMPTY page — an index or careers payload holding no records — from a
@@ -157,7 +160,8 @@ class PageOutcome:
 
     @property
     def ok(self) -> bool:
-        return not self.load_failed and self.blocked_by is None
+        return (not self.load_failed and self.blocked_by is None
+                and self.server_error is None)
 
 
 ITEM_LINK_SELECTOR = page_flow.READY_SELECTOR_LISTING
@@ -1149,6 +1153,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
         return outcome
 
     outcome.state = state
+    outcome.server_error = page_flow.target_server_error(state, http_status)
+    if outcome.server_error is not None:
+        logger.error("Page %d: Mercor answered HTTP %d (a server error) on every "
+                     "attempt. Nothing was obtained from it, so this page is "
+                     "FAILED, not empty.", page_num, outcome.server_error)
+        outcome.final_url = session.page.url
+        return outcome
 
     if state == "blocked":
         # What a caller needs here is WHICH refusal this is, because the two
@@ -1562,8 +1573,7 @@ def scrape(args) -> int:
             outcomes.append(first)
 
             if not first.ok:
-                stop_reason = ("page_load_timeout" if first.load_failed
-                               else f"blocked_{first.blocked_by}")
+                stop_reason = page_flow.failure_stop_reason(first)
                 blocked = first.blocked_by is not None
             elif args.mode == "job" and not job_urls:
                 pass  # `--url` named one job; one page is the whole run
@@ -1600,8 +1610,7 @@ def scrape(args) -> int:
                     failed = [o for o in rest if not o.ok]
                     if failed:
                         worst = min(failed, key=lambda o: o.page_num)
-                        stop_reason = ("page_load_timeout" if worst.load_failed
-                                       else f"blocked_{worst.blocked_by}")
+                        stop_reason = page_flow.failure_stop_reason(worst)
                         blocked = any(o.blocked_by for o in rest)
                     elif exhausted:
                         stop_reason = "no_new_products"
@@ -1623,8 +1632,7 @@ def scrape(args) -> int:
                         outcome = _fetch_one_page(session, args, pool, page_num, url)
                         outcomes.append(outcome)
                         if not outcome.ok:
-                            stop_reason = ("page_load_timeout" if outcome.load_failed
-                                           else f"blocked_{outcome.blocked_by}")
+                            stop_reason = page_flow.failure_stop_reason(outcome)
                             blocked = outcome.blocked_by is not None
                             break
 
@@ -1771,6 +1779,12 @@ def scrape(args) -> int:
         # Pages the site served and this parser could not read — by number,
         # so the loss is a list a reader can act on rather than a count.
         extra["pages_unreadable"] = unreadable_pages
+    http_errors = page_flow.target_http_errors(outcomes)
+    if http_errors:
+        # Pages the TARGET answered with a server error, and its status:
+        # failed pages, recorded with the status so a 503 is not mistaken
+        # for a dead proxy or an empty index.
+        extra["pages_target_http_error"] = http_errors
     if args.mode == "listings":
         extra["records_in_payload"] = total_available
         extra["urls_in_itemlist"] = indexed

puppeteer_scraper.py

@@ -254,6 +254,9 @@ class PageOutcome:
     products: List = field(default_factory=list)
     blocked_by: Optional[str] = None
     load_failed: bool = False
+    # The target's 5xx status when the page ended as a server error after
+    # its retries (see page_flow.target_server_error). Not ok: no content.
+    server_error: Optional[int] = None
     state: Optional[str] = None
     # What this response said about itself. Mercor publishes no catalogue
     # and how many pages the site will serve. These are what pages are
@@ -274,7 +277,8 @@ class PageOutcome:
 
     @property
     def ok(self) -> bool:
-        return not self.load_failed and self.blocked_by is None
+        return (not self.load_failed and self.blocked_by is None
+                and self.server_error is None)
 
 
 # Every `scheme://user:pass@` in a string, however many times it occurs.
@@ -939,6 +943,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
         return outcome
 
     outcome.state = state
+    outcome.server_error = page_flow.target_server_error(state, http_status)
+    if outcome.server_error is not None:
+        logger.error("Page %d: Mercor answered HTTP %d (a server error) on every "
+                     "attempt. Nothing was obtained from it, so this page is "
+                     "FAILED, not empty.", page_num, outcome.server_error)
+        outcome.final_url = page.url
+        return outcome
 
     if state == "blocked":
         # Mercor has never refused this scraper — see
@@ -1149,8 +1160,7 @@ def scrape(args) -> int:
         outcomes.append(first)
 
         if not first.ok:
-            stop_reason = ("page_load_timeout" if first.load_failed
-                           else f"blocked_{first.blocked_by}")
+            stop_reason = page_flow.failure_stop_reason(first)
             blocked = first.blocked_by is not None
         elif args.mode == "job" and not job_urls:
             pass  # `--url` named one job; one page is the whole run
@@ -1175,8 +1185,7 @@ def scrape(args) -> int:
                 outcome = _fetch_one_page(session, args, pool, page_num, url)
                 outcomes.append(outcome)
                 if not outcome.ok:
-                    stop_reason = ("page_load_timeout" if outcome.load_failed
-                                   else f"blocked_{outcome.blocked_by}")
+                    stop_reason = page_flow.failure_stop_reason(outcome)
                     blocked = outcome.blocked_by is not None
                     break
 
@@ -1256,6 +1265,12 @@ def scrape(args) -> int:
         # Pages the site served and this parser could not read — by number,
         # so the loss is a list a reader can act on rather than a count.
         extra["pages_unreadable"] = unreadable_pages
+    http_errors = page_flow.target_http_errors(outcomes)
+    if http_errors:
+        # Pages the TARGET answered with a server error, and its status:
+        # failed pages, recorded with the status so a 503 is not mistaken
+        # for a dead proxy or an empty index.
+        extra["pages_target_http_error"] = http_errors
     if args.mode == "listings":
         extra["records_in_payload"] = total_available
         extra["urls_in_itemlist"] = indexed

scraper_api_client.py

@@ -70,6 +70,11 @@
 # of one exit code is how a family's contract drifts.
 from output_writer import (EXIT_API_ERROR, SOURCE_DEFAULT,  # noqa: E402
                            finish_run)
+from page_flow import TARGET_SERVER_ERROR, target_server_error  # noqa: E402
+
+# `_run_once` asks `main` for another attempt after a target 5xx. Never
+# returned as an exit code: the last attempt reports the failure itself.
+_RETRY_TARGET_5XX = -5
 
 def _mask_credentials(url: str) -> str:
     """Never print a username:password embedded in a ws://... or http://... URL."""
@@ -187,6 +192,11 @@ def fetch_html(args) -> str:
     if not isinstance(upstream_status, int):
         legacy = body.get("status")
         upstream_status = legacy if isinstance(legacy, int) else None
+    if upstream_status is None and body.get("status") == "error":
+        # The API's documented verdict for a target network error or 5xx.
+        # With no target status to classify, nothing was obtained: exit 5.
+        raise RuntimeError("Scraper API task ended with status 'error' and "
+                           "no target status: the page was not obtained")
     logger.info("Upstream page HTTP %s (API status %r), %d bytes of HTML.",
                 upstream_status, body.get("status"), len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
@@ -212,9 +222,10 @@ def main() -> int:
     attempts = max(1, args.retries + 1)
     for attempt in range(1, attempts + 1):
         rc = _run_once(args, attempt, attempts)
-        if rc != 3 or attempt == attempts:
+        if rc not in (3, _RETRY_TARGET_5XX) or attempt == attempts:
             return rc
-        logger.info("Challenge page on attempt %d/%d — retrying in %ds.",
+        logger.info("%s on attempt %d/%d — retrying in %ds.",
+                    "Challenge page" if rc == 3 else "Target server error",
                     attempt, attempts, args.retry_delay)
         time.sleep(args.retry_delay)
     return rc
@@ -267,6 +278,26 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int:
 
     stop_reason = ("single_page_route" if args.mode in ("listings", "careers")
                    else "completed")
+    server_error = target_server_error(state, upstream_status)
+    if server_error is not None:
+        # Mercor's own 5xx, carried in `http_code` of an HTTP 200 API answer.
+        # It says nothing about what the page holds: retried within
+        # --retries, then the content was not obtained (exit 5), never the
+        # empty result of exit 4. No rows, so earlier output is kept.
+        if attempt < attempts:
+            return _RETRY_TARGET_5XX
+        logger.error("Mercor answered HTTP %d for %s on every attempt — "
+                     "exit %d: the page was not obtained, NOT an empty "
+                     "result (exit 4).", server_error, args.url, EXIT_API_ERROR)
+        return finish_run([], args.out, args.format, args.allow_empty,
+                          blocked=False, stop_reason=TARGET_SERVER_ERROR,
+                          pages_requested=1, pages_completed=0,
+                          pages_failed=[1], mode=args.mode,
+                          source=SOURCE_DEFAULT, start_url=args.url,
+                          final_url=args.url,
+                          extra={"route_is_paginated": args.mode == "job",
+                                 "mode": args.mode,
+                                 "pages_target_http_error": {"1": server_error}})
     if state == "not_found":
         # `upstream_status` is the TARGET's status: the Scraper API answers
         # HTTP 200 and carries a site 404 in `http_code` (its documentation

selenium_scraper.py

@@ -157,6 +157,9 @@ class PageOutcome:
     products: List = field(default_factory=list)
     blocked_by: Optional[str] = None
     load_failed: bool = False
+    # The target's 5xx status when the page ended as a server error after
+    # its retries (see page_flow.target_server_error). Not ok: no content.
+    server_error: Optional[int] = None
     state: Optional[str] = None
     # What this response said about itself. Mercor publishes no catalogue
     # and how many pages the site will serve. Real numbers here, unlike the
@@ -177,7 +180,8 @@ class PageOutcome:
 
     @property
     def ok(self) -> bool:
-        return not self.load_failed and self.blocked_by is None
+        return (not self.load_failed and self.blocked_by is None
+                and self.server_error is None)
 
 
 # Every `scheme://user:pass@` in a string, however many times it occurs.
@@ -881,6 +885,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
         return outcome
 
     outcome.state = state
+    outcome.server_error = page_flow.target_server_error(state, http_status)
+    if outcome.server_error is not None:
+        logger.error("Page %d: Mercor answered HTTP %d (a server error) on every "
+                     "attempt. Nothing was obtained from it, so this page is "
+                     "FAILED, not empty.", page_num, outcome.server_error)
+        outcome.final_url = d["current_url"]()
+        return outcome
 
     if state == "blocked":
         # Mercor has never refused this scraper — see
@@ -1088,8 +1099,7 @@ def scrape(args) -> int:
         outcomes.append(first)
 
         if not first.ok:
-            stop_reason = ("page_load_timeout" if first.load_failed
-                           else f"blocked_{first.blocked_by}")
+            stop_reason = page_flow.failure_stop_reason(first)
             blocked = first.blocked_by is not None
         elif args.mode == "job" and not job_urls:
             pass  # `--url` named one job; one page is the whole run
@@ -1114,8 +1124,7 @@ def scrape(args) -> int:
                 outcome = _fetch_one_page(session, args, pool, page_num, url)
                 outcomes.append(outcome)
                 if not outcome.ok:
-                    stop_reason = ("page_load_timeout" if outcome.load_failed
-                                   else f"blocked_{outcome.blocked_by}")
+                    stop_reason = page_flow.failure_stop_reason(outcome)
                     blocked = outcome.blocked_by is not None
                     break
 
@@ -1195,6 +1204,12 @@ def scrape(args) -> int:
         # Pages the site served and this parser could not read — by number,
         # so the loss is a list a reader can act on rather than a count.
         extra["pages_unreadable"] = unreadable_pages
+    http_errors = page_flow.target_http_errors(outcomes)
+    if http_errors:
+        # Pages the TARGET answered with a server error, and its status:
+        # failed pages, recorded with the status so a 503 is not mistaken
+        # for a dead proxy or an empty index.
+        extra["pages_target_http_error"] = http_errors
     if args.mode == "listings":
         extra["records_in_payload"] = total_available
         extra["urls_in_itemlist"] = indexed