CDP: leave a connected browser running, never hang connecting, retry a 500

CHANGELOG.md

@@ -79,6 +79,25 @@ discovered from a bill or a diff.
   matches (`git check-ignore --no-index`). Without that it never reported a
   file already tracked, so it only worked until the fixture was committed.
 
+- **pyppeteer no longer shuts down a browser it connected to.** Over
+  `--cdp-endpoint` its teardown sent CDP `Browser.close`, killing the browser
+  on the other end — a local Chrome, or the session on the Scraping Browser
+  API — and printing `Event loop is closed` tracebacks; the next run then
+  hung. A connected browser now has its page closed and the connection
+  dropped. It also took the remote browser's own first tab and navigated it
+  away; it now opens a new page.
+- **A CDP endpoint that refuses the handshake no longer hangs pyppeteer.**
+  pyppeteer's own `connect()` swallowed a refused port or an HTTP 500 into a
+  background task and waited for an answer that never came. The connection
+  now raises the real reason (`HTTP 500`, `Connection refused`, or a
+  handshake that never completed) within the timeout.
+- **Playwright closes the page it opened over CDP.** One tab was left in the
+  remote browser per run (1 → 2 → 3 over two runs on a local Chrome).
+- **Both engines retry a CDP handshake refused with HTTP 500, 502 or 503**
+  twice, after 3 s and 6 s. A managed browser can refuse for a few seconds
+  while a previous session on the same profile closes. Other refusals —
+  401, a closed port, a timeout — fail at once.
+
 ### Changed
 
 - CI only: the canary now runs weekly (Mondays 06:17 UTC) instead of daily, and on a manual dispatch.

README.md

@@ -207,7 +207,7 @@ package ids were identical.
 | **Playwright** (primary) | `requirements-playwright.txt` | the one to use |
 | Selenium | `requirements-selenium.txt` | no authenticated remote CDP endpoint (`--cdp-endpoint` is refused, exit 2); `--proxy-server` cannot authenticate at all, so credentials are stripped and a warning printed; exposes no HTTP status, so classification leans on the structural signals (Play's own 404 template is recognised from its body); `--fingerprint` is refused |
 | pyppeteer | `requirements-puppeteer.txt` | effectively unmaintained upstream; here for parity; `--fingerprint` is refused |
-| Scraping Browser API | `--cdp-endpoint` | a managed browser; never stack `--proxy` or `--fingerprint` on it |
+| Scraping Browser API | `--cdp-endpoint` | a managed browser; never stack `--proxy` or `--fingerprint` on it. Playwright and pyppeteer open their own page on it and leave the browser running when the run ends; a handshake refused with HTTP 500/502/503 is retried twice (3 s, 6 s), anything else fails at once with exit 5 |
 
 **Install exactly one per virtualenv.** Playwright and pyppeteer declare
 mutually unsatisfiable pins (`pyee` <12 vs ≥13) and pyppeteer and selenium

page_flow.py

@@ -17,6 +17,7 @@
 primitives in and this module names the OPERATION.
 """
 
+import re
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
 import urllib.parse
 
@@ -347,3 +348,50 @@ def block_advice(html: Optional[str], headless: bool,
             "here — but if a reCAPTCHA does appear on a refused page, the "
             "run will solve it with a key (at most one paid solve per page)")
     return tips
+
+
+# --------------------------------------------------------------------------
+# Connecting to a browser over CDP
+# --------------------------------------------------------------------------
+
+# A managed browser can refuse the WebSocket handshake for a few seconds
+# after a previous session on the same profile ended: the 2Captcha Scraping
+# Browser API answers HTTP 500 while it tears the old session down (reported
+# by a manual test of 0.2.0, back-to-back runs). That refusal costs nothing —
+# no session was created — and clears on its own, so it is worth a short wait
+# and another try. A refused port or a timeout is not: the first says nothing
+# is there, the second has already waited the full timeout.
+CDP_CONNECT_ATTEMPTS = 3
+CDP_CONNECT_BACKOFF_S = (3.0, 6.0)
+_CDP_RETRIABLE_STATUSES = (500, 502, 503)
+
+_STATUS_IN_TEXT = re.compile(r"\b(50[023])\b")
+
+
+def cdp_connect_status(exc: BaseException) -> Optional[int]:
+    """The HTTP status a failed CDP handshake carried, if it says.
+
+    `websockets` reports it as `status_code` (10.x) or `response.status_code`
+    (newer); Playwright only in its message — "WebSocket error: <url> 500
+    Internal Server Error", measured against a local server answering 500.
+    """
+    for value in (getattr(exc, "status_code", None),
+                  getattr(getattr(exc, "response", None), "status_code", None)):
+        if isinstance(value, int):
+            return value
+    text = str(exc)
+    if "WebSocket" in text or "HTTP" in text or "handshake" in text.lower():
+        m = _STATUS_IN_TEXT.search(text)
+        if m:
+            return int(m.group(1))
+    return None
+
+
+def cdp_connect_retriable(exc: BaseException) -> bool:
+    return cdp_connect_status(exc) in _CDP_RETRIABLE_STATUSES
+
+
+def cdp_connect_delay(attempt: int) -> float:
+    """Seconds to wait before attempt `attempt + 1` (attempt counts from 1)."""
+    idx = max(0, min(attempt - 1, len(CDP_CONNECT_BACKOFF_S) - 1))
+    return CDP_CONNECT_BACKOFF_S[idx]

playwright_scraper.py

@@ -26,6 +26,7 @@
 """
 
 import sys
+import time
 from typing import Any, Dict, Optional, Tuple
 
 # Imported at MODULE level on purpose: the smoke suite must be able to skip
@@ -46,11 +47,17 @@ class PlaywrightDriver:
 
     name = NAME
 
-    def __init__(self, pw_ctx, browser, context, page) -> None:
+    def __init__(self, pw_ctx, browser, context, page, connected: bool = False,
+                 owns_context: bool = True) -> None:
         self._pw_ctx = pw_ctx
         self._browser = browser
         self._context = context
         self._page = page
+        # Whether the browser was reached over --cdp-endpoint rather than
+        # launched here, and so which parts of it this run may tear down.
+        self._connected = connected
+        self._owns_context = owns_context
+        self._owns_page = True
 
     def fetch(self, url: str, mode: str) -> Tuple[str, Optional[int]]:
         """One navigation.
@@ -95,12 +102,56 @@ def current_url(self) -> str:
         return self._page.url
 
     def close(self) -> None:
+        """Leave the browser the way it was found.
+
+        A browser this engine LAUNCHED is closed. One it CONNECTED to over
+        `--cdp-endpoint` belongs to somebody else: only the page — and the
+        context, if this run created it — are closed, and then the connection
+        is dropped. `Browser.close()` on a CDP connection only disconnects, so
+        the page opened for the run stayed open in the remote browser: one
+        more tab per run (1 -> 2 -> 3 over two runs, measured 2026-10-22 on a
+        local Chrome with --remote-debugging-port).
+        """
         try:
+            if self._connected:
+                for owned in (self._page if self._owns_page else None,
+                              self._context if self._owns_context else None):
+                    if owned is None:
+                        continue
+                    try:
+                        owned.close()
+                    except PlaywrightError:
+                        pass        # already gone with the remote session
             self._browser.close()
         finally:
             self._pw_ctx.__exit__(None, None, None)
 
 
+def _connect_over_cdp(pw, endpoint: str):
+    """`connect_over_cdp`, retrying the handshake refusals worth retrying.
+
+    Playwright reports a refused handshake at once ("WebSocket error: ...
+    500 Internal Server Error"), so the policy in `page_flow` decides whether
+    to wait and try again; anything else is raised to the caller as it was.
+    """
+    attempt = 0
+    while True:
+        attempt += 1
+        try:
+            return pw.chromium.connect_over_cdp(
+                endpoint, timeout=engine_core.DEFAULT_TIMEOUT_MS)
+        except PlaywrightError as exc:
+            if (attempt >= page_flow.CDP_CONNECT_ATTEMPTS
+                    or not page_flow.cdp_connect_retriable(exc)):
+                raise
+            delay = page_flow.cdp_connect_delay(attempt)
+            print(f"[warn] the CDP endpoint refused the connection (HTTP "
+                  f"{page_flow.cdp_connect_status(exc)}); retrying in "
+                  f"{delay:.0f}s ({attempt}/{page_flow.CDP_CONNECT_ATTEMPTS})",
+                  file=sys.stderr)
+            time.sleep(delay)
+
+
 def launch(args, proxy_url: Optional[str]) -> PlaywrightDriver:
     """A browser, a context and a page.
 
@@ -115,10 +166,17 @@ def launch(args, proxy_url: Optional[str]) -> PlaywrightDriver:
         # Never set a UA, a fingerprint or a proxy over CDP: the remote
         # browser brings its own identity and stacking a second one
         # manufactures a contradiction rather than better cover.
-        browser = pw.chromium.connect_over_cdp(
-            args.cdp_endpoint, timeout=engine_core.DEFAULT_TIMEOUT_MS)
-        context = browser.contexts[0] if browser.contexts else browser.new_context()
-        return PlaywrightDriver(pw_ctx, browser, context, context.new_page())
+        try:
+            browser = _connect_over_cdp(pw, args.cdp_endpoint)
+        except BaseException:
+            pw_ctx.__exit__(None, None, None)
+            raise
+        owns_context = not browser.contexts
+        context = browser.new_context() if owns_context else browser.contexts[0]
+        # A NEW page, always: the remote browser's existing tabs are its
+        # owner's, and this run closes only what it opened.
+        return PlaywrightDriver(pw_ctx, browser, context, context.new_page(),
+                                connected=True, owns_context=owns_context)
 
     context_kwargs: Dict[str, Any] = {}
     init_script = None

puppeteer_scraper.py

@@ -51,10 +51,13 @@
 class PuppeteerDriver:
     name = NAME
 
-    def __init__(self, loop, browser, page) -> None:
+    def __init__(self, loop, browser, page, connected: bool = False) -> None:
         self._loop = loop
         self._browser = browser
         self._page = page
+        # Reached over --cdp-endpoint rather than launched here: the browser
+        # is somebody else's, and only the page this run opened is ours.
+        self._connected = connected
 
     def _run(self, coro):
         return self._loop.run_until_complete(coro)
@@ -114,23 +117,148 @@ def current_url(self) -> str:
         return self._page.url or ""
 
     def close(self) -> None:
+        """Leave the browser the way it was found.
+
+        For a CONNECTED browser pyppeteer's `Browser.close()` sends CDP
+        `Browser.close` — it shut down the browser on the other end of
+        `--cdp-endpoint`, whoever's it was, and the next run then hung on a
+        dead endpoint (measured 2026-10-22 on a local Chrome with
+        --remote-debugging-port; reported on the Scraping Browser too). A
+        connected browser now gets its page closed and the connection dropped;
+        only a browser this engine launched is closed.
+
+        The connection's own tasks are then let finish before the event loop
+        is closed. Closing the loop under them printed `Event loop is closed`
+        tracebacks after every CDP run.
+        """
         try:
-            self._run(self._browser.close())
+            if self._connected:
+                try:
+                    self._run(self._page.close())
+                except Exception:   # the session may already be gone
+                    pass
+                self._run(self._browser.disconnect())
+            else:
+                self._run(self._browser.close())
         finally:
-            self._loop.close()
+            _drain_and_close(self._loop)
+
+
+class CDPConnectError(ConnectionError):
+    """The CDP endpoint could not be connected to, with the reason it gave."""
+
+    def __init__(self, message: str, status_code: Optional[int] = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+async def _connect(endpoint: str, timeout_s: float):
+    """pyppeteer's `connect()`, but a refused handshake RAISES.
+
+    pyppeteer opens the WebSocket in a background task and then waits for the
+    answer to its first CDP command. When the handshake fails — a refused
+    port, an HTTP 500 from a managed browser — that background task dies with
+    the reason and the command is never answered, so `connect()` waited
+    forever: the "next run hangs" in the report. Measured 2026-10-22 against
+    a local server answering 500 and against a closed port: both hung, with
+    the real error visible only in a "Task exception was never retrieved"
+    traceback at exit.
+
+    This waits on the first command AND on the background task, and raises
+    whichever finishes first with an error, bounded by `timeout_s`. It relies
+    on `Connection._recv_fut`, a private attribute present in every pyppeteer
+    release this repository supports; without it, it degrades to the timeout.
+    """
+    from pyppeteer.browser import Browser
+    from pyppeteer.connection import Connection
+
+    loop = asyncio.get_event_loop()
+    connection = Connection(endpoint, loop, 0)
+    # `send()` returns a Future, not a coroutine, so `ensure_future` and not
+    # `create_task`.
+    first = asyncio.ensure_future(connection.send("Target.getBrowserContexts"))
+    receiver = getattr(connection, "_recv_fut", None)
+    waiting = {first} | ({receiver} if receiver is not None else set())
+    done, _pending = await asyncio.wait(waiting, timeout=timeout_s,
+                                        return_when=asyncio.FIRST_COMPLETED)
+    if first not in done:
+        reason = None
+        if receiver is not None and receiver in done and not receiver.cancelled():
+            reason = receiver.exception()
+        # Dispose BEFORE touching `first`: it is the very future pyppeteer
+        # keeps for the pending command, and `dispose()` fails it with
+        # "Target closed" — cancelling it first made that an InvalidStateError
+        # that replaced the real reason.
+        try:
+            await connection.dispose()
+        except Exception:
+            pass
+        if first.done() and not first.cancelled():
+            first.exception()       # retrieved, so it is not reported at exit
+        else:
+            first.cancel()
+        if isinstance(reason, (asyncio.TimeoutError, TimeoutError)):
+            raise CDPConnectError(
+                "the CDP endpoint accepted the connection and never completed "
+                "the WebSocket handshake") from None
+        if reason is not None:
+            raise CDPConnectError(
+                f"the CDP endpoint refused the connection: "
+                f"{type(reason).__name__}: {reason}",
+                page_flow.cdp_connect_status(reason)) from None
+        raise CDPConnectError(
+            f"no answer from the CDP endpoint within {timeout_s:.0f}s")
+    contexts = first.result().get("browserContextIds", [])
+    return await Browser.create(connection, contexts, False,
+                                {"width": 800, "height": 600}, None,
+                                lambda: connection.send("Browser.close"))
+
+
+def _connect_with_retry(loop, endpoint: str):
+    timeout_s = engine_core.DEFAULT_TIMEOUT_MS / 1000.0
+    attempt = 0
+    while True:
+        attempt += 1
+        try:
+            return loop.run_until_complete(_connect(endpoint, timeout_s))
+        except CDPConnectError as exc:
+            if (attempt >= page_flow.CDP_CONNECT_ATTEMPTS
+                    or not page_flow.cdp_connect_retriable(exc)):
+                raise
+            delay = page_flow.cdp_connect_delay(attempt)
+            print(f"[warn] the CDP endpoint refused the connection (HTTP "
+                  f"{exc.status_code}); retrying in {delay:.0f}s "
+                  f"({attempt}/{page_flow.CDP_CONNECT_ATTEMPTS})", file=sys.stderr)
+            loop.run_until_complete(asyncio.sleep(delay))
+
+
+def _drain_and_close(loop) -> None:
+    """Let the connection's own tasks finish, then close the loop."""
+    try:
+        pending = [t for t in asyncio.all_tasks(loop) if not t.done()]
+        for task in pending:
+            task.cancel()
+        if pending:
+            loop.run_until_complete(
+                asyncio.gather(*pending, return_exceptions=True))
+    finally:
+        loop.close()
 
 
 def launch(args, proxy_url: Optional[str]) -> PuppeteerDriver:
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
 
     if args.cdp_endpoint:
-        from pyppeteer import connect
-        browser = loop.run_until_complete(
-            connect(browserWSEndpoint=args.cdp_endpoint))
-        pages = loop.run_until_complete(browser.pages())
-        page = pages[0] if pages else loop.run_until_complete(browser.newPage())
-        return PuppeteerDriver(loop, browser, page)
+        try:
+            browser = _connect_with_retry(loop, args.cdp_endpoint)
+            # A NEW page, always. This used to take `pages()[0]` — the
+            # remote browser's own first tab — and navigate it away.
+            page = loop.run_until_complete(browser.newPage())
+        except BaseException:
+            _drain_and_close(loop)
+            raise
+        return PuppeteerDriver(loop, browser, page, connected=True)
 
     launch_args = ["--disable-blink-features=AutomationControlled",
                    "--window-size=1440,2400"]